From 43c63adc255a3a32b89946b20ce441dbd925656e Mon Sep 17 00:00:00 2001 From: Levigos Date: Tue, 17 Apr 2018 11:43:40 +0300 Subject: [PATCH] Updating github to v0.0.3 Theres no vendor folder for now, so it wont build Client and server is now both use this repo, so rip casper-client If you want to see a sheer amount of work was done, just look at diff with previous commit We might want to include some version of our internal changelog, sorry for this time --- .gitignore | 7 +- Godeps/Godeps.json | 2 +- .../briantigerchow/pubsub/pubsub_test.go | 3 +- .../go-random-files/random-files/main.go | 4 +- .../go-random-files/ringreader/ringreader.go | 2 +- .../jbenet/go-random/random/random.go | 3 +- .../levenshtein/levenshtein_test.go | 3 +- ISSUE_TEMPLATE.md | 33 - README.md | 7 +- assets/assets.go | 15 +- blocks/blocks.go | 134 ++++ blocks/blockstore/blockstore.go | 23 +- blocks/blockstore/blockstore_test.go | 3 +- blocks/blockstore/caching.go | 6 +- blocks/blockstore/util/remove.go | 4 +- blocks/blocksutil/block_generator.go | 10 +- blockservice/blockservice.go | 38 +- blockservice/blockservice_test.go | 7 +- blockservice/test/blocks_test.go | 7 +- blockservice/test/mock.go | 10 +- casper/casper_utils/casper_utils.go | 252 ++++-- casper/casper_utils/iputils.go | 101 +++ casper/casper_utils/iputils_test.go | 34 + casper/casper_utils/memory/interproc.go | 11 +- casper/crypto/aes.go | 104 +++ casper/restapi/fileshare.go | 52 ++ casper/restapi/restapi.go | 514 ++++++++++++ casper/restapi/restapi.yaml | 51 ++ casper/thrift/thrift.go | 119 +++ casper/uuid/uuid.go | 33 + casper/uuid/uuid_test.go | 26 + casper/validation/pinger.go | 158 ++++ casper/validation/server.go | 283 +++++++ casper/validation/validation.go | 141 ++++ casper/validation/validation_test.go | 92 +++ client/main.go | 126 +++ cmd/ipfs/.gitignore | 3 - cmd/ipfs/serverdaemon.go | 231 ------ cmd/ipfs/serverhandler.go | 172 ---- cmd/ipfs_client/.gitignore | 3 + cmd/ipfs_client/Rules.mk | 50 ++ cmd/{ipfs => ipfs_client}/daemon.go | 57 +- cmd/{ipfs => ipfs_client}/dist/LICENSE | 0 cmd/{ipfs => ipfs_client}/dist/README.md | 0 cmd/ipfs_client/dist/install.sh | 36 + cmd/{ipfs => ipfs_client}/init.go | 19 +- cmd/ipfs_client/ipfs.go | 100 +++ cmd/ipfs_client/main.go | 733 +++++++++++++++++ cmd/ipfs_client/main_test.go | 14 + cmd/{ipfs => ipfs_client}/runmain_test.go | 0 cmd/ipfs_client/util/ulimit.go | 91 +++ cmd/ipfs_client/util/ulimit_freebsd.go | 27 + cmd/ipfs_client/util/ulimit_test.go | 88 +++ cmd/ipfs_client/util/ulimit_unix.go | 27 + cmd/ipfs_client/util/ulimit_windows.go | 7 + cmd/ipfs_server/.gitignore | 3 + cmd/{ipfs => ipfs_server}/1.txt | 0 cmd/{ipfs => ipfs_server}/Rules.mk | 0 cmd/ipfs_server/daemon.go | 748 ++++++++++++++++++ cmd/ipfs_server/dist/LICENSE | 21 + cmd/ipfs_server/dist/README.md | 28 + cmd/{ipfs => ipfs_server}/dist/install.sh | 0 cmd/ipfs_server/init.go | 260 ++++++ cmd/{ipfs => ipfs_server}/ipfs.go | 4 +- cmd/{ipfs => ipfs_server}/main.go | 42 +- cmd/{ipfs => ipfs_server}/main_test.go | 2 +- cmd/ipfs_server/runmain_test.go | 30 + cmd/{ipfs => ipfs_server}/runwbootstrap.bat | 0 cmd/ipfs_server/serverdaemon.go | 300 +++++++ cmd/ipfs_server/serverhandler.go | 260 ++++++ cmd/{ipfs => ipfs_server}/ulimit.go | 0 cmd/{ipfs => ipfs_server}/ulimit_freebsd.go | 0 cmd/{ipfs => ipfs_server}/ulimit_unix.go | 0 cmd/ipfswatch/ipfswatch_test.go | 2 +- cmd/ipfswatch/main.go | 16 +- commands/cli/cmd_suggestion.go | 4 +- commands/cli/helptext.go | 2 +- commands/cli/helptext_test.go | 2 +- commands/cli/parse.go | 40 +- commands/cli/parse_test.go | 2 +- commands/command.go | 4 +- commands/files/serialfile.go | 15 +- commands/http/client.go | 4 +- commands/http/handler.go | 16 +- commands/http/handler_test.go | 6 +- commands/http/multifilereader.go | 2 +- commands/http/multifilereader_test.go | 2 +- commands/http/parse.go | 6 +- commands/option.go | 9 +- commands/request.go | 7 +- commands/response.go | 1 + core/bootstrap.go | 5 +- core/bootstrap_test.go | 3 +- core/builder.go | 20 +- core/commands/active.go | 2 +- core/commands/add.go | 220 +++++- core/commands/bitswap.go | 6 +- core/commands/block.go | 102 ++- core/commands/bootstrap.go | 9 +- core/commands/cat.go | 65 +- core/commands/commands.go | 2 +- core/commands/config.go | 8 +- core/commands/dag/dag.go | 191 ++++- core/commands/del.go | 81 ++ core/commands/dht.go | 6 +- core/commands/dht_test.go | 3 +- core/commands/diag.go | 2 +- core/commands/dns.go | 5 +- core/commands/external.go | 2 +- core/commands/files/files.go | 14 +- core/commands/filestore.go | 7 +- core/commands/get.go | 70 +- core/commands/helptext_test.go | 2 +- core/commands/id.go | 5 +- core/commands/ipns.go | 7 +- core/commands/keystore.go | 2 +- core/commands/log.go | 2 +- core/commands/ls.go | 18 +- core/commands/mount_nofuse.go | 2 +- core/commands/mount_unix.go | 6 +- core/commands/mount_windows.go | 2 +- core/commands/name.go | 2 +- core/commands/object/diff.go | 8 +- core/commands/object/object.go | 12 +- core/commands/object/patch.go | 13 +- core/commands/p2p.go | 4 +- core/commands/pin.go | 14 +- core/commands/ping.go | 5 +- core/commands/publish.go | 8 +- core/commands/pubsub.go | 5 +- core/commands/refs.go | 8 +- core/commands/repo.go | 12 +- core/commands/resolve.go | 9 +- core/commands/root.go | 17 +- core/commands/shutdown.go | 2 +- core/commands/stat.go | 3 +- core/commands/swarm.go | 9 +- core/commands/sysdiag.go | 8 +- core/commands/tar.go | 12 +- core/commands/unixfs/ls.go | 14 +- core/commands/unixfs/unixfs.go | 2 +- core/commands/upd.go | 99 +++ core/commands/validate.go | 77 ++ core/commands/version.go | 6 +- core/core.go | 214 ++++- core/core_test.go | 7 +- core/coreapi/coreapi.go | 8 +- core/coreapi/unixfs.go | 6 +- core/coreapi/unixfs_test.go | 19 +- core/coredag/dagpb.go | 2 +- core/coredag/raw.go | 2 +- core/corehttp/commands.go | 10 +- core/corehttp/corehttp.go | 3 +- core/corehttp/gateway.go | 7 +- core/corehttp/gateway_handler.go | 20 +- core/corehttp/gateway_indexPage.go | 2 +- core/corehttp/gateway_test.go | 16 +- core/corehttp/ipns_hostname.go | 2 +- core/corehttp/logs.go | 3 +- core/corehttp/metrics.go | 2 +- core/corehttp/metrics_test.go | 2 +- core/corehttp/redirect.go | 2 +- core/corerepo/gc.go | 8 +- core/corerepo/pinning.go | 6 +- core/corerepo/stat.go | 5 +- core/coreunix/add.go | 48 +- core/coreunix/add_test.go | 21 +- core/coreunix/cat.go | 20 +- core/coreunix/metadata.go | 7 +- core/coreunix/metadata_test.go | 18 +- core/mock/mock.go | 11 +- core/pathresolver.go | 4 +- core/pathresolver_test.go | 6 +- doc.go | 27 - exchange/bitswap/bitswap.go | 20 +- exchange/bitswap/bitswap_test.go | 15 +- exchange/bitswap/decision/bench_test.go | 3 +- exchange/bitswap/decision/engine.go | 6 +- exchange/bitswap/decision/engine_test.go | 5 +- exchange/bitswap/decision/ledger.go | 58 +- .../bitswap/decision/peer_request_queue.go | 4 +- .../decision/peer_request_queue_test.go | 3 +- exchange/bitswap/get.go | 5 +- exchange/bitswap/message/message.go | 21 +- exchange/bitswap/message/message_test.go | 3 +- exchange/bitswap/network/interface.go | 2 +- exchange/bitswap/network/ipfs_impl.go | 2 +- .../bitswap/notifications/notifications.go | 3 +- .../notifications/notifications_test.go | 3 +- exchange/bitswap/session.go | 2 +- exchange/bitswap/session_test.go | 2 +- exchange/bitswap/testnet/interface.go | 3 +- exchange/bitswap/testnet/network_test.go | 9 +- exchange/bitswap/testnet/peernet.go | 5 +- exchange/bitswap/testnet/virtual.go | 8 +- exchange/bitswap/testutils.go | 9 +- exchange/bitswap/wantmanager.go | 8 +- exchange/bitswap/workers.go | 2 +- exchange/offline/offline.go | 5 +- exchange/offline/offline_test.go | 5 +- exchange/reprovide/providers.go | 6 +- exchange/reprovide/reprovide_test.go | 7 +- filestore/filestore.go | 5 +- filestore/filestore_test.go | 6 +- filestore/fsrefstore.go | 9 +- filestore/util.go | 6 +- fuse/ipns/common.go | 9 +- fuse/ipns/ipns_test.go | 9 +- fuse/ipns/ipns_unix.go | 12 +- fuse/ipns/mount_unix.go | 4 +- fuse/node/mount_darwin.go | 2 +- fuse/node/mount_nofuse.go | 2 +- fuse/node/mount_test.go | 11 +- fuse/node/mount_unix.go | 9 +- fuse/node/mount_windows.go | 2 +- fuse/readonly/ipfs_test.go | 15 +- fuse/readonly/mount_unix.go | 4 +- fuse/readonly/readonly_unix.go | 10 +- importer/balanced/balanced_test.go | 10 +- importer/balanced/builder.go | 14 +- importer/chunk/rabin.go | 2 +- importer/helpers/dagbuilder.go | 22 +- importer/helpers/helpers.go | 17 +- importer/importer.go | 14 +- importer/importer_test.go | 8 +- importer/trickle/trickle_test.go | 12 +- importer/trickle/trickledag.go | 6 +- merkledag/coding.go | 19 +- merkledag/merkledag.go | 13 +- merkledag/merkledag_test.go | 19 +- merkledag/node.go | 60 +- merkledag/node_test.go | 4 +- merkledag/pb/merkledag.pb.go | 1 + merkledag/raw.go | 21 +- merkledag/test/utils.go | 9 +- merkledag/traverse/traverse_test.go | 4 +- merkledag/utils/diff.go | 2 +- merkledag/utils/diffenum.go | 2 +- merkledag/utils/diffenum_test.go | 4 +- merkledag/utils/utils.go | 10 +- merkledag/utils/utils_test.go | 6 +- mfs/dir.go | 8 +- mfs/fd.go | 2 +- mfs/file.go | 8 +- mfs/mfs_test.go | 18 +- mfs/ops.go | 2 +- mfs/system.go | 4 +- mk/golang.mk | 2 +- namesys/base.go | 2 +- namesys/dns.go | 2 +- namesys/interface.go | 3 +- namesys/ipns_select_test.go | 5 +- namesys/namesys.go | 2 +- namesys/namesys_test.go | 6 +- namesys/proquint.go | 3 +- namesys/publisher.go | 12 +- namesys/publisher_test.go | 6 +- namesys/republisher/repub.go | 10 +- namesys/republisher/repub_test.go | 11 +- namesys/resolve_test.go | 5 +- namesys/routing.go | 4 +- path/resolver.go | 2 +- path/resolver_test.go | 6 +- pin/gc/gc.go | 6 +- pin/pin.go | 4 +- pin/pin_test.go | 8 +- pin/set.go | 4 +- pin/set_test.go | 8 +- plugin/ipld.go | 2 +- plugin/loader/initializer.go | 4 +- plugin/loader/load.go | 2 +- plugin/loader/load_linux.go | 2 +- plugin/loader/preload.go | 2 +- plugin/plugins/git/git.go | 4 +- repo/config/casper.go | 6 +- repo/config/config.go | 6 +- repo/config/init.go | 24 +- repo/config/profile.go | 20 + repo/config/swarm.go | 16 + repo/fsrepo/config_test.go | 2 +- repo/fsrepo/datastores.go | 2 +- repo/fsrepo/fsrepo.go | 22 +- repo/fsrepo/fsrepo_test.go | 5 +- repo/fsrepo/migrations/mfsr_test.go | 2 +- repo/fsrepo/misc.go | 4 +- repo/fsrepo/serialize/serialize.go | 3 +- repo/fsrepo/serialize/serialize_test.go | 2 +- repo/mock.go | 6 +- repo/repo.go | 6 +- routing/mock/centralized_client.go | 3 +- routing/mock/centralized_test.go | 3 +- routing/mock/interface.go | 3 +- routing/none/none_client.go | 2 +- routing/offline/offline.go | 2 +- submodule/go-ipld-cbor/node.go | 573 ++++++++++++++ submodule/go-ipld-cbor/node_test.go | 433 ++++++++++ tar/format.go | 12 +- test/bench/bench_cli_ipfs_add/main.go | 6 +- test/bench/offline_add/main.go | 6 +- test/integration/addcat_test.go | 12 +- test/integration/bench_cat_test.go | 9 +- test/integration/bench_test.go | 3 +- test/integration/bitswap_wo_routing_test.go | 5 +- test/integration/three_legged_cat_test.go | 9 +- test/sharness/lib/random-dep.go | 4 +- thirdparty/datastore2/delayed.go | 2 +- thirdparty/tar/extractor.go | 7 + unixfs/archive/archive.go | 6 +- unixfs/archive/tar/writer.go | 8 +- unixfs/format.go | 9 +- unixfs/format_test.go | 2 +- unixfs/hamt/hamt.go | 6 +- unixfs/hamt/hamt_stress_test.go | 6 +- unixfs/hamt/hamt_test.go | 8 +- unixfs/io/dagreader.go | 6 +- unixfs/io/dagreader_test.go | 6 +- unixfs/io/dirbuilder.go | 7 +- unixfs/io/dirbuilder_test.go | 4 +- unixfs/io/pbdagreader.go | 6 +- unixfs/io/resolve.go | 6 +- unixfs/mod/dagmodifier.go | 12 +- unixfs/mod/dagmodifier_test.go | 8 +- unixfs/test/utils.go | 12 +- 323 files changed, 8271 insertions(+), 1488 deletions(-) delete mode 100644 ISSUE_TEMPLATE.md create mode 100644 blocks/blocks.go create mode 100644 casper/casper_utils/iputils.go create mode 100644 casper/casper_utils/iputils_test.go create mode 100644 casper/crypto/aes.go create mode 100644 casper/restapi/fileshare.go create mode 100644 casper/restapi/restapi.go create mode 100644 casper/restapi/restapi.yaml create mode 100644 casper/thrift/thrift.go create mode 100644 casper/uuid/uuid.go create mode 100644 casper/uuid/uuid_test.go create mode 100644 casper/validation/pinger.go create mode 100644 casper/validation/server.go create mode 100644 casper/validation/validation.go create mode 100644 casper/validation/validation_test.go create mode 100644 client/main.go delete mode 100644 cmd/ipfs/.gitignore delete mode 100644 cmd/ipfs/serverdaemon.go delete mode 100644 cmd/ipfs/serverhandler.go create mode 100644 cmd/ipfs_client/.gitignore create mode 100644 cmd/ipfs_client/Rules.mk rename cmd/{ipfs => ipfs_client}/daemon.go (94%) rename cmd/{ipfs => ipfs_client}/dist/LICENSE (100%) rename cmd/{ipfs => ipfs_client}/dist/README.md (100%) create mode 100644 cmd/ipfs_client/dist/install.sh rename cmd/{ipfs => ipfs_client}/init.go (94%) create mode 100644 cmd/ipfs_client/ipfs.go create mode 100644 cmd/ipfs_client/main.go create mode 100644 cmd/ipfs_client/main_test.go rename cmd/{ipfs => ipfs_client}/runmain_test.go (100%) create mode 100644 cmd/ipfs_client/util/ulimit.go create mode 100644 cmd/ipfs_client/util/ulimit_freebsd.go create mode 100644 cmd/ipfs_client/util/ulimit_test.go create mode 100644 cmd/ipfs_client/util/ulimit_unix.go create mode 100644 cmd/ipfs_client/util/ulimit_windows.go create mode 100644 cmd/ipfs_server/.gitignore rename cmd/{ipfs => ipfs_server}/1.txt (100%) rename cmd/{ipfs => ipfs_server}/Rules.mk (100%) create mode 100644 cmd/ipfs_server/daemon.go create mode 100644 cmd/ipfs_server/dist/LICENSE create mode 100644 cmd/ipfs_server/dist/README.md rename cmd/{ipfs => ipfs_server}/dist/install.sh (100%) create mode 100644 cmd/ipfs_server/init.go rename cmd/{ipfs => ipfs_server}/ipfs.go (97%) rename cmd/{ipfs => ipfs_server}/main.go (95%) rename cmd/{ipfs => ipfs_server}/main_test.go (86%) create mode 100644 cmd/ipfs_server/runmain_test.go rename cmd/{ipfs => ipfs_server}/runwbootstrap.bat (100%) create mode 100644 cmd/ipfs_server/serverdaemon.go create mode 100644 cmd/ipfs_server/serverhandler.go rename cmd/{ipfs => ipfs_server}/ulimit.go (100%) rename cmd/{ipfs => ipfs_server}/ulimit_freebsd.go (100%) rename cmd/{ipfs => ipfs_server}/ulimit_unix.go (100%) create mode 100644 core/commands/del.go create mode 100644 core/commands/upd.go create mode 100644 core/commands/validate.go delete mode 100644 doc.go create mode 100644 submodule/go-ipld-cbor/node.go create mode 100644 submodule/go-ipld-cbor/node_test.go diff --git a/.gitignore b/.gitignore index 190a218..360d618 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,13 @@ coverage.txt +# for docker +libdl.so.2 + +vendor/gitlab.com + .ipfs bin/gx bin/gx* bin/tmp -.idea \ No newline at end of file +.idea diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 7057f35..a04a32f 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,5 +1,5 @@ { - "ImportPath": "github.com/ipfs/go-ipfs", + "ImportPath": "gitlab.com/casperDev/Casper-server", "GoVersion": "go1.5.2", "Packages": [ "./..." diff --git a/Godeps/_workspace/src/github.com/briantigerchow/pubsub/pubsub_test.go b/Godeps/_workspace/src/github.com/briantigerchow/pubsub/pubsub_test.go index f4a80ab..94ef098 100644 --- a/Godeps/_workspace/src/github.com/briantigerchow/pubsub/pubsub_test.go +++ b/Godeps/_workspace/src/github.com/briantigerchow/pubsub/pubsub_test.go @@ -5,10 +5,11 @@ package pubsub import ( - check "gopkg.in/check.v1" "runtime" "testing" "time" + + check "gopkg.in/check.v1" ) var _ = check.Suite(new(Suite)) diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go b/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go index c5af8cb..2b7cb75 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go @@ -9,8 +9,8 @@ import ( "os" "time" - randomfiles "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random-files" - ringreader "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader" + randomfiles "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random-files" + ringreader "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader" ) var usage = `usage: %s [options] ... diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go b/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go index 6b09cd4..c281c1d 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go @@ -5,7 +5,7 @@ import ( "fmt" "math/rand" - random "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" + random "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" ) type Reader struct { diff --git a/Godeps/_workspace/src/github.com/jbenet/go-random/random/random.go b/Godeps/_workspace/src/github.com/jbenet/go-random/random/random.go index 65a9d02..4db5ab2 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-random/random/random.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-random/random/random.go @@ -5,8 +5,9 @@ import ( "os" "strconv" - random "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" + + random "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" ) func main() { diff --git a/Godeps/_workspace/src/github.com/texttheater/golang-levenshtein/levenshtein/levenshtein_test.go b/Godeps/_workspace/src/github.com/texttheater/golang-levenshtein/levenshtein/levenshtein_test.go index 4b68447..e138505 100644 --- a/Godeps/_workspace/src/github.com/texttheater/golang-levenshtein/levenshtein/levenshtein_test.go +++ b/Godeps/_workspace/src/github.com/texttheater/golang-levenshtein/levenshtein/levenshtein_test.go @@ -1,8 +1,9 @@ package levenshtein_test import ( - "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/texttheater/golang-levenshtein/levenshtein" "testing" + + "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/texttheater/golang-levenshtein/levenshtein" ) var testCases = []struct { diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md deleted file mode 100644 index 95a2acf..0000000 --- a/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,33 +0,0 @@ - -#### Version information: - - -#### Type: - - -#### Severity: - -#### Description: - - - - - - - - diff --git a/README.md b/README.md index f5de20f..99cd0b6 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,5 @@ - - - - - +## ATTENTION +As of now we are not providing you folder "vendor", so server won't build, and nor client. Sorry for the inconvenience, we'll think of something the following week. # CasperAPI provider application. CasperAPI is a decentralized data storage. We were deeply concerned with the state of the Internet and thought that with the state-of-art decentralized technologies we could at least start changing the way data is stored and distributed right now. diff --git a/assets/assets.go b/assets/assets.go index 683ecf5..9f91659 100644 --- a/assets/assets.go +++ b/assets/assets.go @@ -9,15 +9,20 @@ import ( "os" "path/filepath" - "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/core/coreunix" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + uid "gitlab.com/casperDev/Casper-server/casper/uuid" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core/coreunix" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" // this import keeps gx from thinking the dep isn't used _ "gx/ipfs/QmdZ4PvPHFQVLLEve7DgoKDcSY19wwpGBB1GKjjKi2rEL1/dir-index-html" ) +var log = logging.Logger("assets") + // initDocPaths lists the paths for the docs we want to seed during --init var initDocPaths = []string{ filepath.Join("init-doc", "about"), @@ -47,12 +52,15 @@ func SeedInitDirIndex(nd *core.IpfsNode) (*cid.Cid, error) { func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) { dirb := uio.NewDirectory(nd.DAG) + log.Debugf("%v", l) for _, p := range l { d, err := Asset(p) if err != nil { return nil, fmt.Errorf("assets: could load Asset '%s': %s", p, err) } + log.Debugf("Few first bytes: %x", d[:uid.UUIDLen]) + // s, err := coreunix.Add(nd, bytes.NewBuffer(append(bl.NullUUID, d...))) s, err := coreunix.Add(nd, bytes.NewBuffer(d)) if err != nil { return nil, fmt.Errorf("assets: could not Add '%s': %s", p, err) @@ -65,6 +73,7 @@ func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) { return nil, err } + log.Debugf("Trying to find CID %s", c.String()) node, err := nd.DAG.Get(nd.Context(), c) if err != nil { return nil, err diff --git a/blocks/blocks.go b/blocks/blocks.go new file mode 100644 index 0000000..4c0bd71 --- /dev/null +++ b/blocks/blocks.go @@ -0,0 +1,134 @@ +// Package blocks implements interface blocks.Block from go-block-format +// augmented with UUID. +// A block is raw data accompanied by a CID. The CID contains the multihash +// corresponding to the block. +package blocks + +import ( + "fmt" + "runtime/debug" + + "gitlab.com/casperDev/Casper-server/casper/uuid" + + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" + mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" +) + +var log = logging.Logger("blocks") + +// A BasicBlock is a singular block of data in ipfs. It implements the Block +// interface. +type BasicBlock struct { + cid *cid.Cid + data []byte + uuid []byte +} + +// SplitData extracts UUID and Data from byte slice +func SplitData(rawdata []byte) (uid []byte, data []byte) { + if len(rawdata) < uuid.UUIDLen { + panic(fmt.Sprintf("Data is too small and has no UUID %x", rawdata)) + } + + return rawdata[:uuid.UUIDLen], rawdata[uuid.UUIDLen:] +} + +// NewBlock creates a Block object from opaque data. It will hash the data. +func NewBlock(data []byte) *BasicBlock { + cid := cid.NewCidV0(u.Hash(data)) + + log.Debugf("NewBlock of length %d and with CID %s", len(data), cid.String()) + + return &BasicBlock{data: data, cid: cid} +} + +func NewBlockWithUUID(data []byte) *BasicBlock { + var id *cid.Cid + + uid, d := SplitData(data) + if uuid.IsUUIDNull(uid) { + uid = nil + id = cid.NewCidV0(u.Hash(data)) + } else { + id = cid.NewCidV0(u.Hash(uid)) + } + + log.Debugf("NewBlockWithUUID of length %d with CID %s", len(d), id.String()) + + // Do not store UUID if it is zero + return &BasicBlock{data: d, uuid: uid, cid: id} +} + +// NewBlockWithCid creates a new block when the hash of the data +// is already known, this is used to save time in situations where +// we are able to be confident that the data is correct. +func NewBlockWithCid(data []byte, c *cid.Cid) (*BasicBlock, error) { + id, d := SplitData(data) + log.Debugf("NewBlockWithCid: splitted uuid %d + %d %s", len(id), len(d), base58.Encode(id)) + + if u.Debug { + var rd = id + if uuid.IsUUIDNull(id) { + rd = d + } + chkc, err := c.Prefix().Sum(rd) + + if err != nil { + return nil, err + } + + if !chkc.Equals(c) { + log.Debugf("Calculated hash %s != Argument hash %s", chkc.String(), c.String()) + debug.PrintStack() + return nil, blocks.ErrWrongHash + } + } + + return &BasicBlock{data: d, uuid: id, cid: c}, nil +} + +// Multihash returns the hash contained in the block CID. +func (b *BasicBlock) Multihash() mh.Multihash { + return b.cid.Hash() +} + +// RawData returns the block raw contents as a byte slice. +func (b *BasicBlock) RawData() []byte { + if b.uuid != nil { + return append(b.uuid, b.data...) + } + + return append(uuid.NullUUID, b.data...) +} + +// UUID returns the uuid of the block +func (b *BasicBlock) UUID() []byte { + if b.uuid != nil { + return b.uuid + } + + return uuid.NullUUID +} + +// Cid returns the content identifier of the block. +func (b *BasicBlock) Cid() *cid.Cid { + return b.cid +} + +// String provides a human-readable representation of the block CID. +func (b *BasicBlock) String() string { + return fmt.Sprintf("[Block %s, UUID: %s]", b.Cid(), base58.Encode(b.UUID())) +} + +// Loggable returns a go-log loggable item. +func (b *BasicBlock) Loggable() map[string]interface{} { + return map[string]interface{}{ + "block": b.Cid().String(), + "data": b.data, + "uuid": b.uuid, + } +} diff --git a/blocks/blockstore/blockstore.go b/blocks/blockstore/blockstore.go index 1f8eb77..c35e506 100644 --- a/blocks/blockstore/blockstore.go +++ b/blocks/blockstore/blockstore.go @@ -8,9 +8,12 @@ import ( "sync" "sync/atomic" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + bl "gitlab.com/casperDev/Casper-server/blocks" + + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" @@ -122,13 +125,16 @@ func (bs *blockstore) Get(k *cid.Cid) (blocks.Block, error) { if err != nil { return nil, err } + bdata, ok := maybeData.([]byte) if !ok { return nil, ErrValueTypeMismatch } + //if false { if bs.rehash { - rbcid, err := k.Prefix().Sum(bdata) + uuid, _ := bl.SplitData(bdata) + rbcid, err := k.Prefix().Sum(uuid) if err != nil { return nil, err } @@ -137,19 +143,20 @@ func (bs *blockstore) Get(k *cid.Cid) (blocks.Block, error) { return nil, ErrHashMismatch } - return blocks.NewBlockWithCid(bdata, rbcid) + return bl.NewBlockWithCid(bdata, k) } - return blocks.NewBlockWithCid(bdata, k) + return bl.NewBlockWithCid(bdata, k) } func (bs *blockstore) Put(block blocks.Block) error { k := dshelp.CidToDsKey(block.Cid()) // Has is cheaper than Put, so see if we already have it - exists, err := bs.datastore.Has(k) - if err == nil && exists { - return nil // already stored. - } + //exists, err := bs.datastore.Has(k) + //if err == nil && exists { + // return nil // already stored. + //} + //return bs.datastore.Put(k, block.RawData()) return bs.datastore.Put(k, block.RawData()) } diff --git a/blocks/blockstore/blockstore_test.go b/blocks/blockstore/blockstore_test.go index 5b2a284..f688a42 100644 --- a/blocks/blockstore/blockstore_test.go +++ b/blocks/blockstore/blockstore_test.go @@ -6,9 +6,10 @@ import ( "fmt" "testing" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/blocks/blockstore/caching.go b/blocks/blockstore/caching.go index 0ea375b..2da41b6 100644 --- a/blocks/blockstore/caching.go +++ b/blocks/blockstore/caching.go @@ -18,9 +18,9 @@ type CacheOpts struct { // DefaultCacheOpts returns a CacheOpts initialized with default values. func DefaultCacheOpts() CacheOpts { return CacheOpts{ - HasBloomFilterSize: 512 << 10, - HasBloomFilterHashes: 7, - HasARCCacheSize: 64 << 10, + HasBloomFilterSize: 0, //512 << 10, + HasBloomFilterHashes: 0, //7, + HasARCCacheSize: 0, //64 << 10, } } diff --git a/blocks/blockstore/util/remove.go b/blocks/blockstore/util/remove.go index cce013a..f27416d 100644 --- a/blocks/blockstore/util/remove.go +++ b/blocks/blockstore/util/remove.go @@ -8,8 +8,8 @@ import ( cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" - bs "github.com/Casper-dev/Casper-server/blocks/blockstore" - "github.com/Casper-dev/Casper-server/pin" + bs "gitlab.com/casperDev/Casper-server/blocks/blockstore" + "gitlab.com/casperDev/Casper-server/pin" ) // RemovedBlock is used to respresent the result of removing a block. diff --git a/blocks/blocksutil/block_generator.go b/blocks/blocksutil/block_generator.go index 2461104..608afc3 100644 --- a/blocks/blocksutil/block_generator.go +++ b/blocks/blocksutil/block_generator.go @@ -2,7 +2,11 @@ // with Blocks. package blocksutil -import "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" +import ( + blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + + bl "gitlab.com/casperDev/Casper-server/blocks" +) // NewBlockGenerator returns an object capable of // producing blocks. @@ -19,9 +23,9 @@ type BlockGenerator struct { } // Next generates a new BasicBlock. -func (bg *BlockGenerator) Next() *blocks.BasicBlock { +func (bg *BlockGenerator) Next() *bl.BasicBlock { bg.seq++ - return blocks.NewBlock([]byte(string(bg.seq))) + return bl.NewBlock([]byte(string(bg.seq))) } // Blocks generates as many BasicBlocks as specified by n. diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go index 0931f7b..896126e 100644 --- a/blockservice/blockservice.go +++ b/blockservice/blockservice.go @@ -8,9 +8,9 @@ import ( "errors" "fmt" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - exchange "github.com/Casper-dev/Casper-server/exchange" - bitswap "github.com/Casper-dev/Casper-server/exchange/bitswap" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + exchange "gitlab.com/casperDev/Casper-server/exchange" + bitswap "gitlab.com/casperDev/Casper-server/exchange/bitswap" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" @@ -99,18 +99,23 @@ func NewSession(ctx context.Context, bs BlockService) *Session { // TODO pass a context into this if the remote.HasBlock is going to remain here. func (s *blockService) AddBlock(o blocks.Block) (*cid.Cid, error) { c := o.Cid() - if s.checkFirst { - has, err := s.blockstore.Has(c) - if err != nil { - return nil, err - } - if has { - return c, nil - } - } + //debug.PrintStack() + log.Debugf("Add Block with CID %s", c.String()) + + ///if s.checkFirst { + /// has, err := s.blockstore.Has(c) + /// if err != nil { + /// return nil, err + /// } + /// if has { + /// log.Debugf("Block with CID %s already exists", c.String()) + /// //return c, nil + /// } + ///} err := s.blockstore.Put(o) + if err != nil { return nil, err } @@ -126,13 +131,12 @@ func (s *blockService) AddBlocks(bs []blocks.Block) ([]*cid.Cid, error) { var toput []blocks.Block if s.checkFirst { for _, b := range bs { - has, err := s.blockstore.Has(b.Cid()) + _, err := s.blockstore.Has(b.Cid()) if err != nil { return nil, err } - if !has { - toput = append(toput, b) - } + + toput = append(toput, b) } } else { toput = bs @@ -157,7 +161,7 @@ func (s *blockService) AddBlocks(bs []blocks.Block) ([]*cid.Cid, error) { // GetBlock retrieves a particular block from the service, // Getting it from the datastore using the key (hash). func (s *blockService) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) { - log.Debugf("BlockService GetBlock: '%s'", c) + //log.Debugf("BlockService GetBlock: %s", c) var f exchange.Fetcher if s.exchange != nil { diff --git a/blockservice/blockservice_test.go b/blockservice/blockservice_test.go index 70f90f2..b6c464f 100644 --- a/blockservice/blockservice_test.go +++ b/blockservice/blockservice_test.go @@ -3,11 +3,12 @@ package blockservice import ( "testing" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - butil "github.com/Casper-dev/Casper-server/blocks/blocksutil" - offline "github.com/Casper-dev/Casper-server/exchange/offline" "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + butil "gitlab.com/casperDev/Casper-server/blocks/blocksutil" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" dssync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" ) diff --git a/blockservice/test/blocks_test.go b/blockservice/test/blocks_test.go index 3469cba..f8ef676 100644 --- a/blockservice/test/blocks_test.go +++ b/blockservice/test/blocks_test.go @@ -7,11 +7,12 @@ import ( "testing" "time" - blockstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - . "github.com/Casper-dev/Casper-server/blockservice" - offline "github.com/Casper-dev/Casper-server/exchange/offline" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + blockstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + . "gitlab.com/casperDev/Casper-server/blockservice" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/blockservice/test/mock.go b/blockservice/test/mock.go index 1d029fc..6196c8b 100644 --- a/blockservice/test/mock.go +++ b/blockservice/test/mock.go @@ -1,11 +1,11 @@ package bstest import ( - . "github.com/Casper-dev/Casper-server/blockservice" - bitswap "github.com/Casper-dev/Casper-server/exchange/bitswap" - tn "github.com/Casper-dev/Casper-server/exchange/bitswap/testnet" - mockrouting "github.com/Casper-dev/Casper-server/routing/mock" - delay "github.com/Casper-dev/Casper-server/thirdparty/delay" + . "gitlab.com/casperDev/Casper-server/blockservice" + bitswap "gitlab.com/casperDev/Casper-server/exchange/bitswap" + tn "gitlab.com/casperDev/Casper-server/exchange/bitswap/testnet" + mockrouting "gitlab.com/casperDev/Casper-server/routing/mock" + delay "gitlab.com/casperDev/Casper-server/thirdparty/delay" ) // Mocks returns |n| connected mock Blockservices diff --git a/casper/casper_utils/casper_utils.go b/casper/casper_utils/casper_utils.go index e730afa..d8ce4e6 100644 --- a/casper/casper_utils/casper_utils.go +++ b/casper/casper_utils/casper_utils.go @@ -1,100 +1,226 @@ package casper_utils import ( - "fmt" + "context" + "fmt" "math/big" - "git.apache.org/thrift.git/lib/go/thrift" - "crypto/tls" - "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-SC/casper_sc" + "net" + "gitlab.com/casperDev/Casper-SC/casper" + "gitlab.com/casperDev/Casper-SC/casper_sc" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/repo/config" - "github.com/Casper-dev/Casper-SC/casperproto" - "github.com/Casper-dev/Casper-server/repo/config" - "regexp" "github.com/ethereum/go-ethereum/core/types" - "time" -) -var fullNodeID string + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" + ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" + "gx/ipfs/QmeS8cCKawUwejVrsBtmC1toTXmwVWZGiRJqzgTURVWeF9/go-ipfs-addr" + "regexp" +) -func RegisterSC(node *core.IpfsNode, cfg *config.Config) { - fmt.Println(node.Identity.Pretty()) - casper, client, auth := Casper_SC.GetSC() - ip := cfg.Casper.IPAddress - re := regexp.MustCompile("/tcp/.+") - port := re.FindString(cfg.Addresses.Swarm[0]) - fullNodeID = "/ip4/" + ip + port + "/ipfs/" + node.Identity.Pretty() - fmt.Println(fullNodeID) +var log = logging.Logger("csp/utils") - ///TODO: debug only: remove as soon as we deploy - addTokenClosure := func() (*types.Transaction, error) { - return casper.AddToken(auth, big.NewInt(int64(cfg.Casper.RepoSize))) - } - Casper_SC.ValidateMineTX(addTokenClosure, client, auth) +type ExternalAddr struct { + IPFSAddr ipfsaddr.IPFSAddr + ThriftAddr net.Addr +} - time.Sleep(1 * time.Second) +var localNode *ExternalAddr - fmt.Println("registering") +func (a *ExternalAddr) Thrift() net.Addr { + return a.ThriftAddr +} +func (a *ExternalAddr) IPFS() ipfsaddr.IPFSAddr { + return a.IPFSAddr +} +func (a *ExternalAddr) String() string { + a.IPFSAddr.String() - registerProviderClosure := func() (*types.Transaction, error) { - return casper.RegisterProvider(auth, fullNodeID, big.NewInt(int64(cfg.Casper.RepoSize))) - } + return a.IPFSAddr.String()[:] +} - Casper_SC.ValidateMineTX(registerProviderClosure, client, auth) +func GetLocalAddr() *ExternalAddr { + return localNode +} - return +func (a *ExternalAddr) ID() string { + return StringGet32Lower(a.IPFSAddr.ID().Pretty()) } +var ErrInvalidLocalAddr = fmt.Errorf("cannot determine IP for SC registration") + +func RegisterSC(node *core.IpfsNode, cfg *config.Config, addresses ...string) error { + if len(addresses) > 0 { + if addr, err := manet.ToNetAddr(ma.StringCast(addresses[0])); err == nil { + + fmt.Println("Thrift external IPs were provided:", addresses, addr) + id, err := ipfsaddr.ParseString(fmt.Sprintf("%s/ipfs/%s", addresses[0], node.Identity.Pretty())) + if err == nil { + fmt.Println("Making local thrift") +/* + fmt.Println("Thrift external IPs were provided:", addresses) + addrS := fmt.Sprintf("%s/ipfs/%s", addr, node.Identity.Pretty()) + if id, err := ipfsaddr.ParseString(addrS); err == nil { +*/ + localNode = &ExternalAddr{id, addr} + } else { + fmt.Println("err while parsing", err) + } + } + } -func GetCasperNodeID() string { - return fullNodeID -} + if localNode == nil { + fmt.Println("No external IPs were provided") + var addr ma.Multiaddr + var ip string + for _, str := range cfg.Addresses.Swarm { + var err error + addr = ma.StringCast(str) + if ip, err = addr.ValueForProtocol(ma.P_IP4); err == nil { + break + } + } + if addr == nil { + log.Error("nil address") + return ErrInvalidLocalAddr + } + iaddr := ma.StringCast("/ipfs/" + node.Identity.Pretty()) + id, err := ipfsaddr.ParseMultiaddr(addr.Encapsulate(iaddr)) + if err != nil { + log.Error("failed to parse multiaddr:", err) + return ErrInvalidLocalAddr + } + localNode = &ExternalAddr{id, &net.TCPAddr{IP: net.ParseIP(ip), Port: 9090}} + } -var protocolFactory = thrift.NewTBinaryProtocolFactoryDefault() + fmt.Println("Full node address:", localNode.String()) -var transportFactory = thrift.NewTBufferedTransportFactory(8192) + //nodeID = StringGet32Lower(node.Identity.Pretty()) + casperClient, client, auth, _ := Casper_SC.GetSC() + + ///TODO: debug only: remove as soon as we deploy + Casper_SC.ValidateMineTX(func() (*types.Transaction, error) { + return casperClient.AddToken(auth, big.NewInt(int64(13370000000))) + }, client, auth) + + udpIp, _, _ := net.SplitHostPort(localNode.Thrift().String()) + connectionString := regexp.MustCompile("ip4/.+/tcp").ReplaceAllString(cfg.Addresses.Swarm[0], "ip4/"+udpIp+"/tcp") + fmt.Println("Conn string", connectionString) + var telegramAddress [32]byte + copy(telegramAddress[:], []byte(cfg.Casper.TelegramAddress)[:31]) + Casper_SC.ValidateMineTX(func() (*types.Transaction, error) { + return casperClient.RegisterProvider(auth, localNode.IPFSAddr.ID().Pretty(), telegramAddress, connectionString, localNode.Thrift().String(), StringGet32Lower(localNode.String()), big.NewInt(int64(13370000000))) + }, client, auth) + Casper_SC.ValidateMineTX(func() (*types.Transaction, error) { + return casperClient.UpdateIpPort(auth, StringGet32Lower(localNode.String()), localNode.Thrift().String()) + }, client, auth) + + return nil +} -var transport thrift.TTransport +func GetNodeConnectionString() string { + return localNode.String() +} -func RunClient(addr string, secure bool) (*casperproto.CasperServerClient, error) { +func StringGet32Lower(inputString string) string { + return inputString[len(inputString)-31:] +} - var err error - if secure { - cfg := new(tls.Config) - cfg.InsecureSkipVerify = true - transport, err = thrift.NewTSSLSocket(addr, cfg) - } else { - transport, err = thrift.NewTSocket(addr) +func GetPeersMultiaddrs(hash string) (ret []ma.Multiaddr, err error) { + caspersclient, _, _, _ := Casper_SC.GetSC() + bytePeers, err := caspersclient.ShowStoringPeers(nil, hash) + if err != nil { + fmt.Println(err) } - fmt.Println("1") + var peers []string + for i := 0; i < len(bytePeers); i++ { + peers = append(peers, string(bytePeers[i][:31])) + } + getMultiaddrsByPeers(peers) + return ret, err +} +func GetPeersMultiaddrsBySize(size int64) (ret []ma.Multiaddr, err error) { + caspersclient, _, _, _ := Casper_SC.GetSC() + peersStruct, err := caspersclient.GetPeers(nil, big.NewInt(size)) if err != nil { - fmt.Println("Error opening socket:", err) - return nil, err + fmt.Println(err) } + peers := []string{peersStruct.Id1, peersStruct.Id2, peersStruct.Id3, peersStruct.Id4} + return getMultiaddrsByPeers(peers) +} - fmt.Println("2") - if transport == nil { - return nil, fmt.Errorf("Error opening socket, got nil transport. Is server available?") +func getMultiaddrsByPeers(peers []string) (ret []ma.Multiaddr, err error) { + caspersclient, _, _, _ := Casper_SC.GetSC() + fmt.Println(peers) + for _, peer := range peers { + func(peer string) { + defer func() { + if r := recover(); r != nil { + fmt.Println("Error with peer", peer, r) + } + }() + ipPort, err := caspersclient.GetUDPIpPort(nil, peer) + hash, err := caspersclient.GetNodeHash(nil, peer) + fmt.Println(ipPort + "/ipfs/" + hash) + if err != nil { + fmt.Println(err) + return + } + if maddr, err := ma.NewMultiaddr(ipPort + "/ipfs/" + hash); err == nil { + ret = append(ret, maddr) + } else { + fmt.Println(err) + } + }(peer) } - fmt.Println("3") - transport, err = transportFactory.GetTransport(transport) - if transport == nil { - return nil, fmt.Errorf("Error from transportFactory.GetTransport(), got nil transport. Is server available?") + return ret, err +} + +func GetPeersMultiaddrsByHash(hash string) (ret []ma.Multiaddr, err error) { + return GetPeersMultiaddrs(hash) +} + +func GetPeersForUpload(size int64) (ret []string) { + //return []string{"10.10.10.1"} + client, _, _, _ := Casper_SC.GetSC() + tx_get, _ := client.GetPeers(nil, big.NewInt(size)) + peers := []string{tx_get.Id1, tx_get.Id2, tx_get.Id3, tx_get.Id4} + + for _, peer := range peers { + if filteredIP, err := client.GetIpPort(nil, peer); err == nil && filteredIP != "" { + ret = append(ret, filteredIP) + } else { + fmt.Println(err) + } } - fmt.Println("4") - err = transport.Open() + return +} + +func GetIpPortsByHash(hash string) (ret []string) { + // FIXME + caspersclient, _, _, _ := Casper_SC.GetSC() + peers, err := caspersclient.ShowStoringPeers(nil, hash) if err != nil { fmt.Println(err) - return nil, err } - fmt.Println("5") + for _, peer := range peers { + ipPort, err := caspersclient.GetIpPort(nil, string(peer[:31])) + if err != nil { + fmt.Println(err) + continue + } + ret = append(ret, ipPort) + } + return +} - return casperproto.NewCasperServerClientFactory(transport, protocolFactory), nil +func SubscribeToVerificationTargetLogs(ctx context.Context, logCallback func(log *casper.CasperVerificationTarget)) { + Casper_SC.SubscribeToReplicationLogs(ctx, logCallback) } -func CloseClient() { - transport.Close() - return +func SubscribeToVerificationConsensusLogs(ctx context.Context, logCallback func(log *casper.CasperConsensusResult)) { + Casper_SC.SubscribeToConsensusLogs(ctx, logCallback) } diff --git a/casper/casper_utils/iputils.go b/casper/casper_utils/iputils.go new file mode 100644 index 0000000..d145b7c --- /dev/null +++ b/casper/casper_utils/iputils.go @@ -0,0 +1,101 @@ +package casper_utils + +import ( + "errors" + "net" + "strconv" + + ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" +) + +// List of reserved IPs by wikipedia +// TODO make this constant +var reservedIPs = []string{ + "0.0.0.0/8", + "10.0.0.0/8", + "100.64.0.0/10", + "127.0.0.0/8", + "169.254.0.0/16", + "172.16.0.0/12", + "192.0.0.0/24", + "192.0.2.0/24", + "192.88.99.0/24", + "192.168.0.0/16", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "240.0.0.0/4", + "255.255.255.255/32", +} + +func IsIPReserved(addr net.IP) bool { + for _, ip := range reservedIPs { + _, ipnet, _ := net.ParseCIDR(ip) + if addr.Mask(ipnet.Mask).Equal(ipnet.IP) { + return true + } + } + return false +} + +func GetLocalIP() (ip string) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "" + } + + ip = "" + for _, addr := range addrs { + switch v := addr.(type) { + case *net.IPNet: + if v.IP.To4() != nil && !v.IP.IsLoopback() { + // if we treffen non-reserved IP, use it + if !IsIPReserved(v.IP) { + return v.IP.String() + } + ip = v.IP.String() + } + } + } + return ip +} + +var ErrMultiaddrWrongFormat = errors.New("Multiaddr must be of form /ip4//tcp/(/...)?") + +func MultiaddrToTCPAddr(maddr ma.Multiaddr) (*net.TCPAddr, error) { + res := ma.Split(maddr) + if len(res) < 2 { + return nil, ErrMultiaddrWrongFormat + } + ip, err := res[0].ValueForProtocol(ma.P_IP4) + if err != nil { + return nil, ErrMultiaddrWrongFormat + } + sp, err := res[1].ValueForProtocol(ma.P_TCP) + if err != nil { + return nil, ErrMultiaddrWrongFormat + } + port, err := strconv.Atoi(sp) + if err != nil { + return nil, ErrMultiaddrWrongFormat + } + return &net.TCPAddr{ + IP: net.ParseIP(ip), + Port: port, + }, nil +} + +func FilterIP(addr string) (ip string) { + a, err := ma.NewMultiaddr(addr) + if err != nil { + return + } + + ip, err = a.ValueForProtocol(ma.P_IP4) + if err != nil { + return + } + + return ip +} diff --git a/casper/casper_utils/iputils_test.go b/casper/casper_utils/iputils_test.go new file mode 100644 index 0000000..bb8d7f0 --- /dev/null +++ b/casper/casper_utils/iputils_test.go @@ -0,0 +1,34 @@ +package casper_utils + +import ( + "net" + "testing" +) + +var ( + classA = []string{"10.0.0.0", "10.255.255.255"} + classB = []string{"172.16.0.0", "172.31.255.255"} + classC = []string{"192.168.0.0", "192.168.255.255"} +) + +func TestIsIPReserved(t *testing.T) { + if ip := "127.0.0.1"; !IsIPReserved(net.ParseIP(ip)) { + t.Errorf("Loopback IP '%s' is reserved", ip) + } + + for _, ip := range classA { + if !IsIPReserved(net.ParseIP(ip)) { + t.Errorf("Class A IP '%s' is reserved", ip) + } + } + for _, ip := range classB { + if !IsIPReserved(net.ParseIP(ip)) { + t.Errorf("Class B IP '%s' is reserved", ip) + } + } + for _, ip := range classC { + if !IsIPReserved(net.ParseIP(ip)) { + t.Errorf("Class C IP '%s' is reserved", ip) + } + } +} diff --git a/casper/casper_utils/memory/interproc.go b/casper/casper_utils/memory/interproc.go index 5473c60..238423a 100644 --- a/casper/casper_utils/memory/interproc.go +++ b/casper/casper_utils/memory/interproc.go @@ -1,12 +1,13 @@ package memory import ( - "net/rpc" - "net" + "fmt" "log" + "net" "net/http" - "github.com/Casper-dev/Casper-server/exchange/bitswap/decision" - "fmt" + "net/rpc" + + "gitlab.com/casperDev/Casper-server/exchange/bitswap/decision" ) type Hashes struct { @@ -38,7 +39,7 @@ func GetRPC() { if err != nil { log.Fatal("arith error:", err) } - fmt.Printf("Arith: %d*%d=%d", reply) + fmt.Printf("Arith: %s", reply) decision.AllowedHashes = reply return } diff --git a/casper/crypto/aes.go b/casper/crypto/aes.go new file mode 100644 index 0000000..da4197d --- /dev/null +++ b/casper/crypto/aes.go @@ -0,0 +1,104 @@ +package crypto + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "io" + + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + + pbkdf2 "golang.org/x/crypto/pbkdf2" +) + +var log = logging.Logger("crypto") + +const ( + AESKeySize = 32 + SaltSize = 16 + iterCount = 10000 +) + +func genSalt() ([]byte, error) { + var salt = make([]byte, SaltSize) + _, err := io.ReadFull(rand.Reader, salt) + if err != nil { + return nil, err + } + + return salt, nil +} + +func getKeyNonce(passwd []byte, salt []byte) (key []byte, nonce []byte) { + tmpbuf := pbkdf2.Key(passwd, salt, iterCount, AESKeySize+aes.BlockSize, sha256.New) + return tmpbuf[:AESKeySize], tmpbuf[AESKeySize:] +} + +func NewAESReaderWithSalt(r io.Reader, passwd []byte, salt []byte) io.Reader { + key, nonce := getKeyNonce(passwd, salt) + c, err := aes.NewCipher(key) + if err != nil { + return nil + } + ctr := cipher.NewCTR(c, nonce) + return cipher.StreamReader{ + S: ctr, + R: r, + } +} + +func NewAESReader(r io.Reader, passwd []byte) io.Reader { + var salt = make([]byte, SaltSize) + _, err := io.ReadFull(r, salt) + if err != nil { + log.Fatal(err) + return nil + } + + key, nonce := getKeyNonce(passwd, salt) + c, err := aes.NewCipher(key) + if err != nil { + log.Fatal(err) + return nil + } + + ctr := cipher.NewCTR(c, nonce) + return cipher.StreamReader{ + S: ctr, + R: r, + } +} + +type aesReadCloser struct { + SR io.Reader + r *io.ReadCloser +} + +func (arc aesReadCloser) Read(p []byte) (int, error) { + return arc.SR.Read(p) +} + +func (arc aesReadCloser) Close() error { + return (*arc.r).Close() +} + +func NewAESEncryptReadCloser(r io.ReadCloser, passwd []byte) io.ReadCloser { + var salt, err = genSalt() + + key, nonce := getKeyNonce(passwd, salt) + c, err := aes.NewCipher(key) + if err != nil { + return nil + } + + ctr := cipher.NewCTR(c, nonce) + return &aesReadCloser{ + SR: io.MultiReader(bytes.NewReader(salt), cipher.StreamReader{ + S: ctr, + R: r, + }), + r: &r, + } +} diff --git a/casper/restapi/fileshare.go b/casper/restapi/fileshare.go new file mode 100644 index 0000000..d31cd91 --- /dev/null +++ b/casper/restapi/fileshare.go @@ -0,0 +1,52 @@ +package restapi + +import ( + "errors" + "fmt" + "net" + "net/http" + "strings" + "sync" + + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core/corehttp" + "gitlab.com/casperDev/Casper-server/path" +) + +const ( + CasperFileSharePath = "/casper/share" +) + +var errLinkNotFound = errors.New("link does not exist or expired") +var fileLinkHandlers = &sync.Map{} + +type fileHandler struct { +} + +func (_ *fileHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + defer recoverHandler() + + log.Debugf("got file request: %v", req.URL.Path) + pth := path.SplitList(strings.TrimPrefix(req.URL.Path, CasperFileSharePath+"/")) + val, ok := fileLinkHandlers.Load(pth[0]) + if !ok || val == nil { + http.Error(w, errLinkNotFound.Error(), http.StatusNotFound) + return + } + h, ok := val.(http.Handler) + if !ok { + http.Error(w, errLinkNotFound.Error(), http.StatusNotFound) + return + } + + // Path need to be relative to shared directory + req.URL.Path = strings.TrimPrefix(req.URL.Path, fmt.Sprintf("%s/%s", CasperFileSharePath, pth[0])) + h.ServeHTTP(w, req) +} + +func CasperFileShareOption() corehttp.ServeOption { + return func(n *core.IpfsNode, l net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + mux.Handle(CasperFileSharePath+"/", &fileHandler{}) + return mux, nil + } +} diff --git a/casper/restapi/restapi.go b/casper/restapi/restapi.go new file mode 100644 index 0000000..43c6a6f --- /dev/null +++ b/casper/restapi/restapi.go @@ -0,0 +1,514 @@ +package restapi + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand" + "mime" + "net" + "net/http" + "runtime/debug" + "strconv" + "strings" + "time" + + blockservice "gitlab.com/casperDev/Casper-server/blockservice" + uuid "gitlab.com/casperDev/Casper-server/casper/uuid" + cmds "gitlab.com/casperDev/Casper-server/commands" + files "gitlab.com/casperDev/Casper-server/commands/files" + cmdsHttp "gitlab.com/casperDev/Casper-server/commands/http" + core "gitlab.com/casperDev/Casper-server/core" + coreCmds "gitlab.com/casperDev/Casper-server/core/commands" + corehttp "gitlab.com/casperDev/Casper-server/core/corehttp" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" +) + +const ( + CasperApiPath = "/casper/v0" + CasperApiFile = "file" + CasperApiShare = "share" + CasperApiStat = "stat" + contentTypeHeader = "Content-Type" + streamHeader = "X-Stream-Output" + contentLengthHeader = "X-Content-Length" + linkExpireTimeout = 5 * time.Minute +) + +var mimeTypes = map[string]string{ + cmds.Protobuf: "application/protobuf", + cmds.JSON: "application/json", + cmds.XML: "application/xml", + cmds.Text: "text/plain", +} + +var log = logging.Logger("csp/api") + +func NewHandler(cctx cmds.Context, root *cmds.Command) http.Handler { + // setup request logger + cctx.ReqLog = new(cmds.ReqLog) + + return &handler{ + cctx: cctx, + root: root, + } +} + +type handler struct { + cctx cmds.Context + root *cmds.Command +} + +type commandOpts struct { + cmdPath []string + opts map[string]interface{} + args []string + file files.File + reader func(cmds.Response) (io.Reader, error) +} + +func recoverHandler() { + if r := recover(); r != nil { + log.Error("a panic has occurred in the commands handler!") + log.Error(r) + + debug.PrintStack() + } +} + +func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + defer recoverHandler() + + pth := path.SplitList(strings.TrimPrefix(req.URL.Path, CasperApiPath+"/")) + if len(pth) > 0 { + switch pth[0] { + case CasperApiFile: + h.processFile(w, req) + return + case CasperApiShare: + h.processShare(w, req) + return + } + } + http.Error(w, "", http.StatusNotFound) + return +} + +func (h *handler) processFile(w http.ResponseWriter, req *http.Request) { + cmdsReq, cmdOpts, err := h.parseRequest(req) + if err == cmdsHttp.ErrNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + fmt.Println(cmdsReq) + + n, err := h.cctx.GetNode() + if err != nil { + http.Error(w, "cant get ipfs node", http.StatusInternalServerError) + return + } + + ctx, cancel := context.WithCancel(n.Context()) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + clientGone := cn.CloseNotify() + go func() { + select { + case <-clientGone: + case <-ctx.Done(): + } + cancel() + }() + } + + cmdsReq.SetInvocContext(h.cctx) + err = cmdsReq.SetRootContext(ctx) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + rlog := h.cctx.ReqLog.Add(cmdsReq) + defer rlog.Finish() + + // call the command + res := h.root.Call(cmdsReq) + + //b, _ := httputil.DumpRequest(req, true) + //fmt.Printf("Request:\n%s\n", string(b)) + + SendResponse(w, req, res, cmdsReq, cmdOpts) +} + +func randString(n int) string { + letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + s := make([]rune, n) + for i, l := 0, len(letters); i < n; i++ { + s[i] = letters[rand.Intn(l)] + } + return string(s) +} + +const magicLength = 8 + +func genMagic() (magic string) { + ok := true + for ok { + magic = randString(magicLength) + _, ok = fileLinkHandlers.Load(magic) + } + return magic +} + +var errNotShareRequest = fmt.Errorf("not a share request") + +func (h *handler) processShare(w http.ResponseWriter, req *http.Request) { + pth := getRequestPath(req) + if len(pth) < 2 { + http.Error(w, "", http.StatusNotFound) + return + } + if req.Method != http.MethodPost { + http.Error(w, "", http.StatusBadRequest) + return + } + + n, err := h.cctx.GetNode() + if err != nil { + http.Error(w, "cant get ipfs node", http.StatusInternalServerError) + return + } + + // We need to take name of first link because we always wrap files + // in directory + id := uuid.UUIDToCid(b58.Decode(pth[1])) + bserv := blockservice.New(n.Blockstore, offline.Exchange(n.Blockstore)) + dserv := dag.NewDAGService(bserv) + node, err := dserv.Get(req.Context(), id) + if err != nil || node == nil || len(node.Links()) != 1 { + http.Error(w, "file not found", http.StatusNotFound) + return + } + + magic := genMagic() + handler := http.FileServer(http.Dir(fmt.Sprintf("/ipfs/%s", id.String()))) + share := fmt.Sprintf("%s/%s/%s", CasperFileSharePath, magic, node.Links()[0].Name) + fileLinkHandlers.Store(magic, handler) + time.AfterFunc(linkExpireTimeout, func() { + log.Debugf("link '%s' has expired", share) + fileLinkHandlers.Delete(magic) + }) + + log.Debugf("file was shared at '%s'", share) + w.WriteHeader(http.StatusOK) + err = cmdsHttp.FlushCopy(w, strings.NewReader(share)) + if err != nil { + log.Error(err) + } +} + +func getRequestPath(req *http.Request) []string { + return path.SplitList(strings.TrimPrefix(req.URL.Path, CasperApiPath+"/")) +} + +func (h *handler) parseRequest(req *http.Request) (cmds.Request, *commandOpts, error) { + if !strings.HasPrefix(req.URL.Path, CasperApiPath) { + return nil, nil, errors.New("Unexpected path prefix") + } + + pth := getRequestPath(req) + for k, v := range req.URL.Query() { + fmt.Printf("%s: %s\n", k, v) + } + + var opts *commandOpts + var err error + switch req.Method { + case http.MethodPost: + opts, err = getAddNewFileOpts(req) + case http.MethodGet: + if len(pth) >= 3 && pth[2] == CasperApiStat { + opts, err = getFileStatOpts(req) + } else { + opts, err = getGetFileOpts(req) + } + case http.MethodPut: + opts, err = getReplaceFileOpts(req) + case http.MethodDelete: + opts, err = getDeleteFileOpts(req) + default: + err = fmt.Errorf("unsupported method") + } + + if err != nil { + return nil, nil, err + } + + optDefs, err := h.root.GetOptions(opts.cmdPath) + if err != nil { + return nil, nil, err + } + + // ignore error because command exists + cmd, _ := h.root.Get(opts.cmdPath[:len(opts.cmdPath)-1]) + cmdsReq, err := cmds.NewRequest(opts.cmdPath, opts.opts, opts.args, opts.file, cmd, optDefs) + if err != nil { + return nil, nil, err + } + + err = cmd.CheckArguments(cmdsReq) + if err != nil { + return nil, nil, err + } + + return cmdsReq, opts, nil +} + +func getAddNewFileOpts(req *http.Request) (*commandOpts, error) { + f, err := getFile(req) + if err != nil { + // file argument is mandatory + return nil, err + } + return &commandOpts{ + cmdPath: []string{"add"}, + opts: map[string]interface{}{ + cmds.EncLong: cmds.JSON, + cmds.CallerOpt: cmds.CallerOptWeb, + "quiet": true, + "uuid": b58.Encode(uuid.GenUUID()), + }, + args: []string{}, + file: f, + reader: func(res cmds.Response) (io.Reader, error) { + // If command was add we must return only one response + // TODO: another method of determining what command we were executing + outChan, _ := res.Output().(<-chan interface{}) + var out io.Reader = bytes.NewReader([]byte{}) + for obj := range outChan { + ao := obj.(*coreunix.AddedObject) + log.Debugf("obj %s '%s' received from add", ao.UUID, ao.Name) + if ao.Name == coreCmds.RootObjectName { + if b, err := json.Marshal(ao); err == nil { + out = bytes.NewReader(b) + } + break + } + } + return out, nil + }, + }, nil +} + +func getReplaceFileOpts(req *http.Request) (*commandOpts, error) { + f, err := getFile(req) + if err != nil { + // file argument is mandatory + return nil, err + } + pth := getRequestPath(req) + return &commandOpts{ + cmdPath: []string{"add"}, + opts: map[string]interface{}{ + cmds.EncLong: cmds.JSON, + cmds.CallerOpt: cmds.CallerOptWeb, + "quiet": true, + "uuid": pth[1], + }, + args: []string{}, + file: f, + reader: func(res cmds.Response) (io.Reader, error) { + // If command was add we must return only one response + // TODO: another method of determining what command we were executing + outChan, _ := res.Output().(<-chan interface{}) + var out io.Reader = bytes.NewReader([]byte{}) + for obj := range outChan { + ao := obj.(*coreunix.AddedObject) + log.Debugf("obj %s '%s' received from add", ao.UUID, ao.Name) + if ao.Name == coreCmds.RootObjectName { + if b, err := json.Marshal(ao); err == nil { + out = bytes.NewReader(b) + } + break + } + } + return out, nil + }, + }, nil +} + +func getGetFileOpts(req *http.Request) (*commandOpts, error) { + pth := getRequestPath(req) + if len(pth) < 2 { + return nil, fmt.Errorf("name is not specified") + } + a, ok := req.URL.Query()["archive"] + archive := ok && len(a) == 1 && a[0] == "1" + cmdPath := []string{"cat"} + if archive { + cmdPath = []string{"get"} + } + return &commandOpts{ + cmdPath: cmdPath, + opts: map[string]interface{}{ + cmds.EncLong: cmds.JSON, + cmds.CallerOpt: cmds.CallerOptWeb, + }, + args: []string{getHash(pth[1])}, + }, nil +} + +func getFileStatOpts(req *http.Request) (*commandOpts, error) { + pth := getRequestPath(req) + return &commandOpts{ + cmdPath: []string{"dag", "stat"}, + opts: map[string]interface{}{ + cmds.EncLong: cmds.JSON, + cmds.CallerOpt: cmds.CallerOptWeb, + }, + args: []string{getHash(pth[1])}, + }, nil +} + +func getDeleteFileOpts(req *http.Request) (*commandOpts, error) { + cmdPath := []string{"del"} + + pth := getRequestPath(req) + return &commandOpts{ + cmdPath: cmdPath, + opts: map[string]interface{}{ + cmds.EncLong: cmds.JSON, + cmds.CallerOpt: cmds.CallerOptWeb, + }, + args: []string{getHash(pth[1])}, + }, nil +} + +func getFile(req *http.Request) (files.File, error) { + ct := req.Header.Get(contentTypeHeader) + mediatype, _, _ := mime.ParseMediaType(ct) + reader, err := req.MultipartReader() + if err != nil { + return nil, err + } + return &files.MultipartFile{ + Mediatype: mediatype, + Reader: reader, + }, nil +} + +func getHash(id string) string { + if u := b58.Decode(id); len(u) == uuid.UUIDLen { + h := uuid.UUIDToHash(u).B58String() + return h + } + return id +} + +func guessMimeType(res cmds.Response) (string, error) { + // Try to guess mimeType from the encoding option + enc, found, err := res.Request().Option(cmds.EncShort).String() + if err != nil { + return "", err + } + if !found { + return "", errors.New("no encoding option set") + } + + if m, ok := mimeTypes[enc]; ok { + return m, nil + } + + return mimeTypes[cmds.JSON], nil +} + +func SendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req cmds.Request, cmdOpts *commandOpts) { + h := w.Header() + // Expose our agent to allow identification + //h.Set("Server", "go-ipfs/"+config.CurrentVersionNumber) + + mime, err := guessMimeType(res) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + status := http.StatusOK + // if response contains an error, write an HTTP error status code + if e := res.Error(); e != nil { + if e.Code == cmds.ErrClient { + status = http.StatusBadRequest + } else { + status = http.StatusInternalServerError + } + // NOTE: The error will actually be written out by the reader below + } + + var out io.Reader + if cmdOpts.reader == nil { + out, err = res.Reader() + } else { + out, err = cmdOpts.reader(res) + } + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Set up our potential trailer + h.Set("Trailer", cmdsHttp.StreamErrHeader) + + if res.Length() > 0 { + h.Set(contentLengthHeader, strconv.FormatUint(res.Length(), 10)) + } + + if _, ok := res.Output().(io.Reader); ok { + // set streams output type to text to avoid issues with browsers rendering + // html pages on priveleged api ports + mime = "text/plain" + h.Set(streamHeader, "1") + } + + // catch-all, set to text as default + if mime == "" { + mime = "text/plain" + } + + h.Set(contentTypeHeader, mime) + + // set 'allowed' headers + //h.Set("Access-Control-Allow-Headers", AllowedExposedHeaders) + // expose those headers + //h.Set("Access-Control-Expose-Headers", AllowedExposedHeaders) + + if r.Method == "HEAD" { // after all the headers. + return + } + + w.WriteHeader(status) + err = cmdsHttp.FlushCopy(w, out) + if err != nil { + log.Error("err: ", err) + w.Header().Set(cmdsHttp.StreamErrHeader, cmdsHttp.SanitizedErrStr(err)) + } +} + +func CasperOption(cctx cmds.Context) corehttp.ServeOption { + return func(n *core.IpfsNode, l net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + cmdHandler := NewHandler(cctx, coreCmds.Root) + mux.Handle(CasperApiPath+"/", cmdHandler) + return mux, nil + } +} diff --git a/casper/restapi/restapi.yaml b/casper/restapi/restapi.yaml new file mode 100644 index 0000000..d4563e7 --- /dev/null +++ b/casper/restapi/restapi.yaml @@ -0,0 +1,51 @@ +GetFile: # получение файла + method: GET + path: /casper/v0/file/name + params: + - name: base58-encoded UUID or HASH + - archive: if 1 then return file as tar-archive + response: + - success: file contents + - error: error text + +AddFile: # добавление файла + method: POST + path: /casper/v0/file + response: + - success: + - Name: always "" + - UUID: file UUID + - Hash: file HASH + - Size: size of raw data + - error: error text + +PutFile: # замена файла + method: PUT + path: /casper/v0/file/name + - name: base58-encoded UUID or HASH + response: + - success: + - Name: always "" + - UUID: file UUID + - Hash: file HASH + - Size: size of raw data + - error: error text + +DeleteFile: # удаление файла + method: DELETE + path: /casper/v0/file/name + - name: base58-encoded UUID or HASH + response: + - error: error text + +FileStat: # информация о файле + method: GET + path: /casper/v0/file/name/stat + - name: base58-encoded UUID or HASH + response: + - success: + - Name: filename + - UUID: file UUID + - Hash: file HASH + - Size: size of raw data + - error: error text \ No newline at end of file diff --git a/casper/thrift/thrift.go b/casper/thrift/thrift.go new file mode 100644 index 0000000..dfad886 --- /dev/null +++ b/casper/thrift/thrift.go @@ -0,0 +1,119 @@ +package thrift + +import ( + "crypto/tls" + "fmt" + "time" + + "gitlab.com/casperDev/Casper-thrift/casperproto" + + "git.apache.org/thrift.git/lib/go/thrift" +) + +type ThriftOpts struct { + TransportFactory thrift.TTransportFactory + ProtocolFactory thrift.TProtocolFactory + Secure bool + Timeout time.Duration +} + +type ThriftClient struct { + *casperproto.CasperServerClient +} + +var defaultThriftOpts = ThriftOpts{ + ProtocolFactory: thrift.NewTBinaryProtocolFactoryDefault(), + TransportFactory: thrift.NewTBufferedTransportFactory(8192), + Secure: false, + Timeout: thriftDefaultTimeout, +} + +const thriftDefaultTimeout = 30 * time.Second + +type ClientFunc func(*ThriftClient) (interface{}, error) + +func RunClientClosure(addr string, cb ClientFunc) (interface{}, error) { + opts := defaultThriftOpts + + var transport thrift.TTransport + var err error + if opts.Secure { + cfg := &tls.Config{InsecureSkipVerify: true} + transport, err = thrift.NewTSSLSocketTimeout(addr, cfg, opts.Timeout) + } else { + transport, err = thrift.NewTSocketTimeout(addr, opts.Timeout) + } + if err != nil { + return nil, err + } + + transport, err = opts.TransportFactory.GetTransport(transport) + if err != nil { + return nil, err + } + + err = transport.Open() + if err != nil { + return nil, err + } + defer transport.Close() + + return cb(&ThriftClient{CasperServerClient: casperproto.NewCasperServerClientFactory(transport, opts.ProtocolFactory)}) +} + +func RunClientDefault(addr string) (*casperproto.CasperServerClient, thrift.TTransport, error) { + return RunClient(addr, defaultThriftOpts) +} + +func RunClient(addr string, opts ThriftOpts) (client *casperproto.CasperServerClient, transport thrift.TTransport, err error) { + if opts.Secure { + cfg := &tls.Config{InsecureSkipVerify: true} + transport, err = thrift.NewTSSLSocketTimeout(addr, cfg, opts.Timeout) + } else { + transport, err = thrift.NewTSocketTimeout(addr, opts.Timeout) + } + if err != nil { + return nil, nil, err + } + + transport, err = opts.TransportFactory.GetTransport(transport) + if err != nil { + return nil, nil, err + } + + err = transport.Open() + if err != nil { + return nil, nil, err + } + return casperproto.NewCasperServerClientFactory(transport, opts.ProtocolFactory), transport, nil +} + +func RunServerDefault(addr string, handler casperproto.CasperServer) error { + return RunServer(addr, handler, defaultThriftOpts) +} + +func RunServer(addr string, handler casperproto.CasperServer, opts ThriftOpts) error { + var transport thrift.TServerTransport + var err error + if opts.Secure { + cfg := new(tls.Config) + if cert, err := tls.LoadX509KeyPair("server.crt", "server.key"); err == nil { + cfg.Certificates = append(cfg.Certificates, cert) + } else { + return err + } + transport, err = thrift.NewTSSLServerSocketTimeout(addr, cfg, opts.Timeout) + } else { + transport, err = thrift.NewTServerSocketTimeout(addr, opts.Timeout) + } + if err != nil { + return err + } + + processor := casperproto.NewCasperServerProcessor(handler) + server := thrift.NewTSimpleServer4(processor, transport, opts.TransportFactory, opts.ProtocolFactory) + + fmt.Printf("Starting the simple server on %s ...\n", addr) + + return server.Serve() +} diff --git a/casper/uuid/uuid.go b/casper/uuid/uuid.go new file mode 100644 index 0000000..f00d2ff --- /dev/null +++ b/casper/uuid/uuid.go @@ -0,0 +1,33 @@ +package uuid + +import ( + "bytes" + + uuid "github.com/satori/go.uuid" + + "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" +) + +const UUIDLen = 16 + +var NullUUID = make([]byte, UUIDLen) + +func IsUUIDNull(u []byte) bool { + return u == nil || bytes.Equal(u, NullUUID) +} + +func GenUUID() []byte { + u, _ := uuid.NewV4() + return u.Bytes() +} + +func UUIDToHash(uuid []byte) multihash.Multihash { + hash, _ := multihash.Sum(uuid, multihash.SHA2_256, -1) + return hash +} + +func UUIDToCid(uuid []byte) *cid.Cid { + mhash, _ := multihash.Sum(uuid, multihash.SHA2_256, -1) + return cid.NewCidV0(mhash) +} diff --git a/casper/uuid/uuid_test.go b/casper/uuid/uuid_test.go new file mode 100644 index 0000000..0bbb93b --- /dev/null +++ b/casper/uuid/uuid_test.go @@ -0,0 +1,26 @@ +package uuid_test + +import ( + "testing" + + "gitlab.com/casperDev/Casper-server/casper/uuid" + + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" +) + +func TestUUIDToHash(t *testing.T) { + testCases := []struct{ uuid, hash string }{ + {"CTxVEi21hGKWz1ptGH7DUJ", "QmW6xj9bDgNBapeaW4WVuXEvLpriGLGbdVHtRmwQwwdVkK"}, + {"SyBiYPJo2yMFG9bJr45DBs", "QmXWZSRFViFme2o7oagVRKr97jBaxNrFvQ6NUffC8ytu4P"}, + } + + for _, c := range testCases { + hash, err := uuid.UUIDToHash(base58.Decode(c.uuid)) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + if hash != c.hash { + t.Fatalf("Expected: %s, got %s", c.hash, hash) + } + } +} diff --git a/casper/validation/pinger.go b/casper/validation/pinger.go new file mode 100644 index 0000000..ea9f9a2 --- /dev/null +++ b/casper/validation/pinger.go @@ -0,0 +1,158 @@ +package validation + +import ( + "regexp" + "fmt" + "math/big" + "time" + "gitlab.com/casperDev/Casper-SC/casper_sc" + "context" + "github.com/ethereum/go-ethereum/core/types" + "gitlab.com/casperDev/Casper-server/casper/casper_utils" + "math/rand" + "net" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "gitlab.com/casperDev/Casper-SC/casper" + "gitlab.com/casperDev/Casper-server/casper/thrift" +) + +type Pinger struct { + casperclient *casper.Casper + client *ethclient.Client + auth *bind.TransactOpts +} + +func (pinger *Pinger) RunPinger(ctx context.Context) { + for { + pinger.casperclient, pinger.client, pinger.auth, _ = Casper_SC.GetSC() + + getPingResult := func() (*types.Transaction, error) { + return pinger.casperclient.GetPingTarget(pinger.auth, casper_utils.GetLocalAddr().ID()) + } + //_, err = casperclient.WatchReturnString(nil, sink) + + pingResultData := Casper_SC.ValidateMineTX(getPingResult, pinger.client, pinger.auth) + hashRegex := regexp.MustCompile("([a-zA-Z0-9]+)") + hash := hashRegex.FindString(pingResultData) + log.Info("waiting for event") + + log.Info("got node !!", hash, "!!") + pinger.checkNodeByHash(ctx, hash) + + fmt.Println(time.Duration(2 +(rand.Int()%7)) * time.Minute) + time.Sleep(time.Duration(2 +(rand.Int()%7)) * time.Minute) + ///fmt.Println(time.Duration(60-time.Now().Minute()+(rand.Int()%59)) * time.Minute) + ///time.Sleep(time.Duration(60-time.Now().Minute()+(rand.Int()%59)) * time.Minute) + } +} + +func (pinger *Pinger) checkNodeByHash(ctx context.Context, hash string) (err error) { + ipRet, err := pinger.casperclient.GetIpPort(nil, hash) + if err != nil { + log.Error(err) + } + var success = false + log.Info("got IP", ipRet) + + if hash == casper_utils.GetLocalAddr().ID() { + log.Info("Pinging self") + //return As of now, it's ok to ping self + } + + ip, _, err := net.SplitHostPort(ipRet) + if err != nil { + log.Error(err) + } + if len(casper_utils.GetLocalAddr().Thrift().String()) < 7 { + success = true ///It's not everyone's problem if current ip is wrong + } else if len(ip) >= 7 { + subnet := regexp.MustCompile("(\\d+\\.){2}").FindString(ip) + currentSubnet := regexp.MustCompile("(\\d+\\.){2}").FindString(casper_utils.GetLocalAddr().Thrift().String()) + if subnet == currentSubnet && subnet != "" { + success = true ///We won't ping nodes from our subnet + } else { + success = pinger.pingNode(ctx, ipRet, hash) + } + } + + log.Info("is", hash, "validation succeed?", success) + resultData := Casper_SC.ValidateMineTX( + func() (*types.Transaction, error) { + return pinger.casperclient.SendPingResult(pinger.auth, hash, success) + }, + pinger.client, + pinger.auth) + re := regexp.MustCompile("Banned!") + log.Info("res data ", resultData) + if re.FindString(resultData) != "" { + log.Info("Go go replication~!") + go startReplication(hash, pinger.casperclient) + } + return +} + +func (pinger *Pinger) pingNode(ctx context.Context, ip string, hash string) (success bool) { + log.Info("pinging", ip) + timestamp := callPing(ctx, ip) + log.Info("timestamp received", timestamp) + success = true + if timestamp == 0 { + success = false + } + return +} + +func callPing(ctx context.Context, ip string) (timestamp int64) { + defer func() { + if r := recover(); r != nil { + log.Info("Recovered when calling ping with", r) + } + }() + ts, _ := thrift.RunClientClosure(ip, func(thriftClient *thrift.ThriftClient) (interface{}, error) { + return thriftClient.Ping(ctx) + }) + return ts.(int64) +} + +func startReplication(hash string, casperclient *casper.Casper) { + + // We get number of banned provider files + n, _ := casperclient.GetNumberOfFiles(nil, hash) + + for i := int64(0); i < n.Int64(); i++ { + + // For each file we get its hash and size + hash, size, err := casperclient.GetFile(nil, hash, big.NewInt(i)) + if err != nil { + log.Error(err) + } + log.Info("hs", hash, size) + // Search for a new peer to store the file + go replicateFile(hash, hash, size, casperclient) + } +} + +func replicateFile(hash string, blockedAddress string, size *big.Int, casperclient *casper.Casper) (ret string) { + defer func() { + if re := recover(); re != nil { + log.Error(re) + } + }() + + peers, err := casperclient.GetPeers(nil, size) + if err != nil { + log.Error(err) + } + // GetPeers returns 4 ips, but we need only one of them + ipPort, err := casperclient.GetIpPort(nil, peers.Id1) + if err != nil { + log.Error(err) + } + if ipPort != "" { + thrift.RunClientClosure(ipPort, func(thriftClient *thrift.ThriftClient) (interface{}, error) { + return thriftClient.SendReplicationQuery(context.TODO(), hash, blockedAddress, size.Int64()) + }) + } + return +} diff --git a/casper/validation/server.go b/casper/validation/server.go new file mode 100644 index 0000000..1f7ea02 --- /dev/null +++ b/casper/validation/server.go @@ -0,0 +1,283 @@ +package validation + +import ( + "context" + "fmt" + "math/rand" + "net" + "sync" + "time" + + cu "gitlab.com/casperDev/Casper-server/casper/casper_utils" + thrift "gitlab.com/casperDev/Casper-server/casper/thrift" + uid "gitlab.com/casperDev/Casper-server/casper/uuid" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/repo/fsrepo" + "gitlab.com/casperDev/Casper-thrift/casperproto" + + node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" + "gx/ipfs/QmeS8cCKawUwejVrsBtmC1toTXmwVWZGiRJqzgTURVWeF9/go-ipfs-addr" + //"gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" +) + +type RunningCheck struct { + info *casperproto.ChunkInfo + round *round + nodes []*cu.ExternalAddr + results map[string]string + mtx sync.Mutex +} + +func NewRunningCheck(cinfo *casperproto.ChunkInfo) *RunningCheck { + return &RunningCheck{ + info: cinfo, + mtx: sync.Mutex{}, + nodes: make([]*cu.ExternalAddr, 0, NumChunkStoringNodes), + results: make(map[string]string, NumChunkStoringNodes), + } +} + +// round is like sync.Cond but also performs Broadcast after specified timeout +type round struct { + cond *sync.Cond + timer *time.Timer + result interface{} +} + +func newRound(timeout time.Duration) *round { + r := &round{cond: sync.NewCond(&sync.Mutex{})} + r.timer = time.AfterFunc(timeout, r.Broadcast) + return r +} + +func (r *round) Wait() { + r.cond.Wait() +} + +func (r *round) Broadcast() { + if r.timer != nil && !r.timer.Stop() { + <-r.timer.C + } + r.cond.Broadcast() +} + +var uuidProvMap = &sync.Map{} + +const NumChunkStoringNodes = 2 + +//const sendChunkInfoTimeout = 1 * time.Minute +const sendChunkInfoTimeout = 10 * time.Second +const sendChecksumTimeout = 10 * time.Second +const sendVerificationQueryTimeout = 10 * time.Second +const round1Timeout = 5 * time.Minute + +const validateBlockSize int64 = 1024 +const diffuseLength = 16 + +func PerformValidation(ctx context.Context, n *core.IpfsNode, uuid string) error { + rc := NewRunningCheck(nil) + uuidProvMap.Store(uuid, rc) + defer uuidProvMap.Delete(uuid) + + node, err := n.DAG.Get(ctx, uid.UUIDToCid(base58.Decode(uuid))) + if err != nil { + return err + } + + rc.info, err = getRandomChunk(ctx, n, node, uuid, validateBlockSize) + if err != nil { + return err + } + + log.Debugf("Random chunk %v %d %d", rc.info.UUID, rc.info.First, rc.info.Last) + + localAddr := cu.GetLocalAddr() + rc.nodes = append(rc.nodes, localAddr) + rc.info.Providers = append(rc.info.Providers, &casperproto.NodeInfo{IpfsAddr: localAddr.String(), ThriftAddr: localAddr.Thrift().String()}) + + uuidProvMap.Store(uuid, rc) + defer uuidProvMap.Delete(uuid) + + log.Info("Sleep for %s", sendVerificationQueryTimeout) + time.Sleep(sendVerificationQueryTimeout) + log.Debugf("Nodes after sleep: %+v", rc.nodes) + + // TODO: Call SC and initiate checking of specified uuid + + // Perform SendChunkInfo calls on all providers + //rc.round = newRound(sendChunkInfoTimeout) + wg := &sync.WaitGroup{} + for _, prov := range rc.nodes[1:] { + log.Debugf("SendChunkInfo to %s", prov) + wg.Add(1) + go func(addr string) { + defer wg.Done() + thrift.RunClientClosure(addr, func(c *thrift.ThriftClient) (interface{}, error) { + return nil, c.SendChunkInfo(ctx, rc.info) + }) + }(prov.Thrift().String()) + } + wg.Wait() + //rc.round.Wait() + + // Send all nodes checksum + salt := getSalt(rc.info.Diffuse, n.Identity.String()) + cs, err := ChecksumSalt(ctx, node, rc.info.First, rc.info.Last, n.DAG, salt) + log.Debugf("Calculated checksum:", cs.B58String()) + //rc.round = newRound(sendChecksumTimeout) + for _, prov := range rc.nodes { + log.Debugf("SendChecksumHash to %s", prov) + go func(addr string) { + thrift.RunClientClosure(addr, func(c *thrift.ThriftClient) (interface{}, error) { + return nil, c.SendChecksumHash(ctx, rc.info.UUID, localAddr.String(), cs.B58String()) + }) + }(prov.Thrift().String()) + } + time.Sleep(sendChecksumTimeout) + //rc.round.Wait() + + // Wait until results all results arrive + //rc.round = newRound(round1Timeout) + // Perform a call to SC + + return nil +} + +func RegisterUUIDProvider(uuid string, ipfsAddr ipfsaddr.IPFSAddr, tAddr net.Addr) { + log.Debugf("Dumping current UUID map") + uuidProvMap.Range(func(k, v interface{}) bool { + log.Debugf("%s: %s", k, v) + return true + }) + if v, ok := uuidProvMap.Load(uuid); ok { + log.Debugf("Store UUID %s, addr %s %s", ipfsAddr, tAddr) + rc := v.(*RunningCheck) + rc.mtx.Lock() + + rc.nodes = append(rc.nodes, &cu.ExternalAddr{ipfsAddr, tAddr}) + ninfo := casperproto.NodeInfo{IpfsAddr: ipfsAddr.String(), ThriftAddr: tAddr.String()} + rc.info.Providers = append(rc.info.Providers, &ninfo) + if len(rc.info.Providers) == NumChunkStoringNodes { // all nodes except current + log.Debugf("Received info about UUID %s for %d nodes", uuid, NumChunkStoringNodes) + //rc.round.Broadcast() + } + fmt.Println(rc.info.Providers) + + rc.mtx.Unlock() + } +} + +func AddRound1Result(ctx context.Context, uuid string, ipfsAddr ipfsaddr.IPFSAddr, hashDiffuse string) { + if v, ok := uuidProvMap.Load(uuid); ok { + rc := v.(*RunningCheck) + rc.mtx.Lock() + + log.Debugf("Old results: %+v", rc.results) + rc.results[ipfsAddr.ID().Pretty()] = hashDiffuse + log.Debugf("New results: %+v", rc.results) + if len(rc.results) == NumChunkStoringNodes { + // rc.round.Broadcast() + } + + rc.mtx.Unlock() + } else { + log.Debugf("No value stored for UUID %s", uuid) + } +} + +func CollectResultsAndRespond(ctx context.Context, cinfo *casperproto.ChunkInfo, configRoot string) { + rc := NewRunningCheck(cinfo) + uuidProvMap.Store(cinfo.UUID, rc) + defer uuidProvMap.Delete(cinfo.UUID) + + repo, err := fsrepo.Open(configRoot) + if err != nil { + return + } + log.Debugf("Got repo") + + n, err := core.NewNode(ctx, &core.BuildCfg{Online: false, Repo: repo}) + if err != nil { + return + } + log.Debugf("Got core node") + + id := uid.UUIDToCid(base58.Decode(cinfo.UUID)) + node, err := n.DAG.Get(ctx, id) + if err != nil { + return + } + log.Debugf("Got ipld node") + + salt := getSalt(cinfo.Diffuse, n.Identity.String()) + cs, err := ChecksumSalt(ctx, node, cinfo.First, cinfo.Last, n.DAG, salt) + log.Debugf("Checksum:", cs.B58String()) + if err != nil { + return + } + + for _, prov := range cinfo.Providers { + addr, err := ipfsaddr.ParseString(prov.IpfsAddr) + if err != nil { + log.Error(err) + continue + } + taddr, err := net.ResolveTCPAddr("tcp", prov.ThriftAddr) + if err != nil { + log.Error(err) + continue + } + rc.nodes = append(rc.nodes, &cu.ExternalAddr{addr, taddr}) + } + + rc.results[n.Identity.Pretty()] = cs.B58String() + log.Debugf("Sleep chunk info: %s", sendChunkInfoTimeout) + time.Sleep(sendChunkInfoTimeout) + + localAddr := cu.GetLocalAddr() + //rc.round = newRound(sendChecksumTimeout) + for _, prov := range rc.nodes { + go func(addr string) { + thrift.RunClientClosure(addr, func(c *thrift.ThriftClient) (interface{}, error) { + return nil, c.SendChecksumHash(ctx, cinfo.UUID, localAddr.IPFS().String(), cs.B58String()) + }) + }(prov.Thrift().String()) + } + //rc.round.Wait() + time.Sleep(sendChecksumTimeout) + + // TODO send only to first + thrift.RunClientClosure(rc.nodes[0].Thrift().String(), func(c *thrift.ThriftClient) (interface{}, error) { + return nil, c.SendValidationResults(ctx, cinfo.UUID, localAddr.String(), rc.results) + }) +} + +func getRandomChunk(ctx context.Context, n *core.IpfsNode, node node.Node, uuid string, blocksize int64) (*casperproto.ChunkInfo, error) { + size, err := GetFilesize(node) + if err != nil { + return nil, err + } + + rind := rand.Int63n((int64(size) / blocksize) + 1) + first := rind * blocksize + last := (rind + 1) * blocksize + if last > int64(size) { + last = int64(size) + } + + diffuse := make([]byte, diffuseLength) + rand.Read(diffuse) + return &casperproto.ChunkInfo{ + UUID: uuid, + First: first, + Last: last, + Providers: make([]*casperproto.NodeInfo, 0, NumChunkStoringNodes), + Diffuse: string(diffuse), + }, nil +} + +func getSalt(a, b string) []byte { + return []byte{} + return []byte(a + b) +} diff --git a/casper/validation/validation.go b/casper/validation/validation.go new file mode 100644 index 0000000..86fb93c --- /dev/null +++ b/casper/validation/validation.go @@ -0,0 +1,141 @@ +package validation + +import ( + "context" + "errors" + + bl "gitlab.com/casperDev/Casper-server/blocks" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + + node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" +) + +var log = logging.Logger("validator") + +var ErrInvalidBoundaries = errors.New("Invalid boundaries") +var ErrNotFileNode = errors.New("Not a DAG file node") + +func min(a, b uint64) uint64 { + if a < b { + return a + } + return b +} + +const CSFuncCode = mh.SHA2_256 + +func CalcChecksum(data, salt []byte) mh.Multihash { + cs, _ := mh.Sum(append(data, salt...), CSFuncCode, -1) + return cs +} + +func GetFilesize(n node.Node) (uint64, error) { + switch v := n.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(v.Data()) + if err != nil { + return 0, err + } + return fsn.FileSize(), nil + case *dag.RawNode: + _, data := bl.SplitData(v.RawData()) + return uint64(len(data)), nil + default: + return 0, ErrNotFileNode + } +} + +func GetSlice(ctx context.Context, n node.Node, start, stop uint64, serv node.NodeGetter) ([]byte, error) { + if start > stop && stop != 0 { + return nil, ErrInvalidBoundaries + } + switch v := n.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(v.Data()) + if err != nil { + return nil, ErrNotFileNode + } + if fsn.Type == ft.TDirectory && len(fsn.Data) == 0 && len(v.Links()) == 1 { + // this directory is wrapped over one file + // return slice of that file + child, err := v.Links()[0].GetNode(ctx, serv) + if err != nil { + return nil, err + } + return GetSlice(ctx, child, start, stop, serv) + } + if fsn.Type != ft.TFile && fsn.Type != ft.TRaw { + return nil, ErrNotFileNode + } + if stop > fsn.FileSize() { + return nil, ErrInvalidBoundaries + } else if stop == 0 { + stop = fsn.FileSize() + } + var buf []byte + if l := uint64(len(fsn.Data)); l > 0 { + log.Debugf("data node: len=%d, #links=%d", l, len(v.Links())) + if stop < l { + return fsn.Data[start:stop], nil + } + // TODO: check: data nodes have no links + return fsn.Data[start:], nil + } + for i, bs := range fsn.BlockSizes() { + log.Debugf("%02d: blocksize=%d, start=%d, stop=%d", i, bs, start, stop) + if start >= bs { + start -= bs + stop -= bs + continue + } + child, err := v.Links()[i].GetNode(ctx, serv) + if err != nil { + return nil, err + } + sl, err := GetSlice(ctx, child, start, min(stop, bs), serv) + if err != nil { + return nil, err + } + log.Debugf("got slice: len=%d:", len(sl)) + buf = append(buf, sl...) + if stop <= bs { + break + } + start = 0 + stop -= bs + } + return buf, nil + case *dag.RawNode: + _, data := bl.SplitData(v.RawData()) + l := uint64(len(data)) + if start < 0 || stop > l { + return nil, ErrInvalidBoundaries + } + if stop == 0 { + stop = l + } + return data[start:stop], nil + } + return nil, ErrNotFileNode +} + +func Checksum(ctx context.Context, n node.Node, start, stop int64, serv node.NodeGetter) (mh.Multihash, error) { + return ChecksumSalt(ctx, n, start, stop, serv, nil) +} +func ChecksumSalt(ctx context.Context, n node.Node, start, stop int64, serv node.NodeGetter, salt []byte) (mh.Multihash, error) { + if start < 0 || stop < 0 { + return nil, ErrInvalidBoundaries + } + + data, err := GetSlice(ctx, n, uint64(start), uint64(stop), serv) + if err != nil { + return nil, err + } + if l := len(data); l != 0 { + log.Debugf("Received data: %d fst=%x lst=%x", l, data[:1], data[l-1:]) + } + return CalcChecksum(data, salt), nil +} diff --git a/casper/validation/validation_test.go b/casper/validation/validation_test.go new file mode 100644 index 0000000..59b67fe --- /dev/null +++ b/casper/validation/validation_test.go @@ -0,0 +1,92 @@ +package validation_test + +import ( + "bytes" + "context" + "testing" + + "fmt" + + bstest "gitlab.com/casperDev/Casper-server/blockservice/test" + "gitlab.com/casperDev/Casper-server/casper/validation" + imp "gitlab.com/casperDev/Casper-server/importer" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + dag "gitlab.com/casperDev/Casper-server/merkledag" +) + +func slicesEqual(a, b []byte) (bool, string) { + l := len(a) + if l != len(b) { + return false, fmt.Sprintf("Slice's lenths differ: %d != %d", len(a), len(b)) + } + for i, c := range a { + if c != b[i] { + return false, fmt.Sprintf("Slices differ at byte %d: %02d != %02d", i, c, b[i]) + } + } + + return true, "" +} + +func TestGetSlice(t *testing.T) { + bsi := bstest.Mocks(1) + ds := dag.NewDAGService(bsi[0]) + + bsize := uint64(512) + l := 2048 * int(bsize) // 1 MiB + + var buf bytes.Buffer + buf.Grow(l) + for i := 0; i < l; i++ { + buf.WriteByte(byte(i % 0x100)) + } + fullb := buf.Bytes() + + // if there will be unexpected errors, try use bytes.NewReader(fullb) + file, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(&buf, int64(bsize))) + if err != nil { + t.Fatal(err) + } + + // Get full file + s, _ := validation.GetSlice(context.Background(), file, 0, 0, ds) + if eq, r := slicesEqual(s, fullb); !eq { + t.Fatalf(r) + } + + // Get slice containing multiple blocks from the middle + s, err = validation.GetSlice(context.Background(), file, bsize*10, bsize*12, ds) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + if eq, r := slicesEqual(s, fullb[:bsize*2]); !eq { + t.Fatalf(r) + } + + // Get slice containing block boundary + s, _ = validation.GetSlice(context.Background(), file, bsize*3-2, bsize*3+2, ds) + if eq, r := slicesEqual(s, []byte{0xFE, 0xFF, 0x00, 0x01}); !eq { + t.Fatalf(r) + } + + // Provide wrong boundaries + _, err = validation.GetSlice(context.Background(), file, 2, 1, ds) + if err != validation.ErrInvalidBoundaries { + t.Fatalf("Unexpected error: %s", err) + } + + // provide file length explicitly + s, err = validation.GetSlice(context.Background(), file, 0, uint64(l), ds) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + if eq, r := slicesEqual(s, fullb); !eq { + t.Fatalf(r) + } + + // provide extra length + _, err = validation.GetSlice(context.Background(), file, 0, uint64(l+1), ds) + if err != validation.ErrInvalidBoundaries { + t.Fatalf("Unexpected error: %s", err) + } +} diff --git a/client/main.go b/client/main.go new file mode 100644 index 0000000..6c479ad --- /dev/null +++ b/client/main.go @@ -0,0 +1,126 @@ +package client + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "context" + "encoding/json" + _ "expvar" + "fmt" + _ "net/http/pprof" + + thrift "gitlab.com/casperDev/Casper-server/casper/thrift" + + "gitlab.com/casperDev/Casper-SC/casper_sc" + + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + + "github.com/ethereum/go-ethereum/core/types" +) + +var log = logging.Logger("client/handler") + +func recoverHandler(name string, args ...interface{}) { + if r := recover(); r != nil { + fmt.Printf("recovered in %s(%s): %s\n", name, args, r) + } +} + +func HandleClientUpload(ctx context.Context, ip string, hash string, size int64, ipList []string) (err error) { + log.Infof("started upload(%s, %s, %d)", ip, hash, size) + defer recoverHandler("upload", ip, hash, size) + + val, err := json.Marshal(ipList) + if err != nil { + return err + } + + _, err = thrift.RunClientClosure(ip, func(c *thrift.ThriftClient) (interface{}, error) { + return c.SendUploadQuery(ctx, hash, string(val), size) + }) + + fmt.Println("upload()") + return err +} + +func HandleClientDownload(ctx context.Context, ip string, hash string, wallet string) (err error) { + log.Infof("started download(%s, %s, %s)", ip, hash, wallet) + defer recoverHandler("download", ip, hash, wallet) + + _, err = thrift.RunClientClosure(ip, func(c *thrift.ThriftClient) (interface{}, error) { + return c.SendDownloadQuery(ctx, hash, "2", wallet) + }) + if err != nil { + return err + } + + casper, sclient, auth, _ := Casper_SC.GetSC() + fmt.Println("Got SC") + //TODO: Change ConfirmDownload call location + Casper_SC.ValidateMineTX(func() (tx *types.Transaction, err error) { + return casper.ConfirmDownload(auth) + }, sclient, auth) + + fmt.Println("download()") + return nil +} + +func HandleClientDelete(ctx context.Context, ip string, hash string) (err error) { + log.Infof("started delete(%s, %s)", ip, hash) + defer recoverHandler("delete", ip, hash) + + _, err = thrift.RunClientClosure(ip, func(c *thrift.ThriftClient) (interface{}, error) { + return c.SendDeleteQuery(ctx, hash) + }) + if err != nil { + return err + } + + fmt.Println("delete()") + return err +} + +func HandleClientUpdate(ctx context.Context, ip string, uuid string, hash string, size int64) (err error) { + log.Infof("started update(%s, %s, %s)", ip, uuid, hash) + defer recoverHandler("update", ip, uuid, hash, size) + + h, err := thrift.RunClientClosure(ip, func(c *thrift.ThriftClient) (interface{}, error) { + return c.SendUpdateQuery(ctx, uuid, hash, size) + }) + if err != nil { + return err + } + + log.Infof("finished update() with %s", h.(string)) + return nil +} + +func InvokeGetFileChecksum(ctx context.Context, ip string, uuid string, first, last int64, salt string) (string, error) { + log.Infof("started GetFileChecksum(%s, %d, %d, %s)", uuid, first, last, salt) + defer recoverHandler("GetFileChecksum", uuid, first, last, salt) + + h, err := thrift.RunClientClosure(ip, func(c *thrift.ThriftClient) (interface{}, error) { + return c.GetFileChecksum(ctx, uuid, first, last, salt) + }) + + log.Infof("Hash: %s, Error: %s", h, err) + + return h.(string), err +} diff --git a/cmd/ipfs/.gitignore b/cmd/ipfs/.gitignore deleted file mode 100644 index 9393c5f..0000000 --- a/cmd/ipfs/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -ipfs -ipfs-test-cover -ipfs.exe diff --git a/cmd/ipfs/serverdaemon.go b/cmd/ipfs/serverdaemon.go deleted file mode 100644 index 8dd7f77..0000000 --- a/cmd/ipfs/serverdaemon.go +++ /dev/null @@ -1,231 +0,0 @@ -package main - -import ( - "context" - "crypto/tls" - "flag" - "fmt" - "math/rand" - "regexp" - "strings" - "time" - - "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-SC/casper_sc" - "github.com/Casper-dev/Casper-server/casper/casper_utils" - "github.com/Casper-dev/Casper-SC/casperproto" - "github.com/ethereum/go-ethereum/core/types" - "github.com/Casper-dev/Casper-server/repo/fsrepo" - - "git.apache.org/thrift.git/lib/go/thrift" - - ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" - "math/big" - "github.com/Casper-dev/Casper-SC/casper" -) - -var currentNode *core.IpfsNode - -func serve(configRoot string) int { - fmt.Println("serving") - ///framed := flag.Bool("framed", false, "Use framed transport") - ///buffered := flag.Bool("buffered", false, "Use buffered transport") - ///TODO: to use a custom port we need to find way for other providers to get that port - var thriftIP string = "0.0.0.0" - - if thriftIP == "" && configRoot != "" { - cfg, err := fsrepo.ConfigAt(configRoot) - if err != nil { - panic(err) - } - - // TODO: Find out if API can be not IP4 or thrift can use IP6 - // may be add new option in config for thrift? - thriftIP, _ = ma.StringCast(cfg.Addresses.API).ValueForProtocol(ma.P_IP4) - } - - if thriftIP == "" { - thriftIP = Casper_SC.GetIPv4() - } - - addr := flag.String("addr", thriftIP+":9090", "Address to listen to") - secure := flag.Bool("secure", false, "Use tls secure transport") - - flag.Parse() - - protocolFactory := thrift.NewTBinaryProtocolFactoryDefault() - transportFactory := thrift.NewTBufferedTransportFactory(8192) - - ///if *framed { - /// transportFactory = thrift.NewTFramedTransportFactory(transportFactory) - ///} - - go runPinger() - - // always run server here - if err := runServer(transportFactory, protocolFactory, *addr, *secure); err != nil { - fmt.Println("error running server:", err) - } - return 0 -} - -func runServer(transportFactory thrift.TTransportFactory, protocolFactory thrift.TProtocolFactory, addr string, secure bool) error { - var transport thrift.TServerTransport - var err error - if secure { - cfg := new(tls.Config) - if cert, err := tls.LoadX509KeyPair("server.crt", "server.key"); err == nil { - cfg.Certificates = append(cfg.Certificates, cert) - } else { - return err - } - transport, err = thrift.NewTSSLServerSocket(addr, cfg) - } else { - transport, err = thrift.NewTServerSocket(addr) - } - - if err != nil { - return err - } - - fmt.Printf("%T\n", transport) - handler := NewCasperServerHandler() - processor := casperproto.NewCasperServerProcessor(handler) - server := thrift.NewTSimpleServer4(processor, transport, transportFactory, protocolFactory) - - fmt.Println("Starting the simple server... on ", addr) - return server.Serve() -} - -var thriftTimeout = 30 - -func runPinger() { - - //time.Sleep(time.Duration(60-time.Now().Minute()) * time.Second) - //fmt.Println(time.Duration(60-time.Now().Minute()) * time.Second) - time.Sleep(time.Duration(30) * time.Second) - fmt.Println(time.Duration(30) * time.Second) - - for { - casperclient, client, auth := Casper_SC.GetSC() - fmt.Println("auth nonce", auth.Nonce) - nonce, _ := client.PendingNonceAt(context.Background(), auth.From) - fmt.Println("pending nonce", nonce) - //sink := make(chan *casper.CasperReturnString) - - getPingResult := func() (*types.Transaction, error) { - return casperclient.GetPingTarget(auth, casper_utils.GetCasperNodeID()) - } - //_, err = casperclient.WatchReturnString(nil, sink) - - nonce, _ = client.PendingNonceAt(context.Background(), auth.From) - fmt.Println("pending nonce", nonce) - - rer := regexp.MustCompile("(/ip4/[/\\d\\w.]+)") - ipRet := strings.TrimSpace(Casper_SC.ValidateMineTX(getPingResult, client, auth)) - fmt.Println("waiting for event") - //retString := <-sink - //fmt.Println("event ret ", retString.Val) - - re := regexp.MustCompile("/.+?/(.+?)/") - fmt.Println("got ip !!", ipRet, "!!") - ipRet = rer.FindString(ipRet) - fmt.Println("got ip !!", ipRet, "!!") - fmt.Println(len(ipRet)) - ips := re.FindStringSubmatch(ipRet) - if len(ips) >= 1 { - filteredIp := ips[1] - fmt.Println("pinging ", filteredIp) - - timestamp := callPing(filteredIp + ":9090") - fmt.Println(timestamp) - success := true - if timestamp < time.Now().Unix()-int64(thriftTimeout) { - success = false - } - - sendPingResultClosure := func() (*types.Transaction, error) { - return casperclient.SendPingResult(auth, ipRet, success) - } - - nonce, _ = client.PendingNonceAt(context.Background(), auth.From) - fmt.Println("pending nonce", nonce) - resultData := Casper_SC.ValidateMineTX(sendPingResultClosure, client, auth) - - ///TODO: change to working regex - re := regexp.MustCompile("Banned!") - fmt.Println("res data ", resultData) - if re.FindString(resultData) != "" { - fmt.Println("Go go replication~!") - go startReplication(ipRet, casperclient) - } - } - - fmt.Println(time.Duration(10+rand.Int()%20) * time.Second) - time.Sleep(time.Duration(10+rand.Int()%20) * time.Second) - ///fmt.Println(time.Duration(60-time.Now().Minute()+(rand.Int()%59)) * time.Second) - ///time.Sleep(time.Duration(60-time.Now().Minute()+(rand.Int()%59)) * time.Second) - } -} - -func callPing(ip string) (timestamp int64) { - defer func() { - if r := recover(); r != nil { - fmt.Println("Recovered in f", r) - } - }() - tclient, _ := casper_utils.RunClient(ip, false) - timestamp, _ = tclient.Ping(context.Background()) - casper_utils.CloseClient() - return -} - -func startReplication(ip string, casperclient *casper.Casper) { - - // We get number of banned provider files - n, _ := casperclient.GetNumberOfFiles(nil, ip) - - for i := int64(0); i < n.Int64(); i++ { - - // For each file we get its hash and size - hash, size, err := casperclient.GetFile(nil, ip, big.NewInt(i)) - if err != nil { - fmt.Println(err) - } - fmt.Println("hs", hash, size) - // Search for a new peer to store the file - go replicateFile(hash, size, casperclient) - } -} - -func FilterIP(addr string) (ret string) { - re := regexp.MustCompile("/.+?/(.+?)/") - if len(re.FindStringSubmatch(addr)) > 1 { - ret = re.FindStringSubmatch(addr)[1] - fmt.Println("Getting " + re.FindStringSubmatch(addr)[1]) - } else { - fmt.Println("Wrong ip") - } - return -} -func replicateFile(hash string, size *big.Int, casperclient *casper.Casper) (ret string) { - defer func() { - if re := recover(); re != nil { - fmt.Println(re) - } - }() - - tx, err := casperclient.GetPeers(nil, size) - if err != nil { - fmt.Println(err) - } - // GetPeers returns 4 ips, but we need only one of them - filteredIP := FilterIP(tx.Ip1) - - if filteredIP != "" { - tclient, _ := casper_utils.RunClient(filteredIP+":9090", false) - tclient.SendReplicationQuery(context.Background(), hash, filteredIP+":9090", size.Int64()) - casper_utils.CloseClient() - } - return -} diff --git a/cmd/ipfs/serverhandler.go b/cmd/ipfs/serverhandler.go deleted file mode 100644 index 367139b..0000000 --- a/cmd/ipfs/serverhandler.go +++ /dev/null @@ -1,172 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "time" - "io" - "github.com/Casper-dev/Casper-SC/casper_sc" - "math/big" - "github.com/Casper-dev/Casper-server/exchange/bitswap/decision" - - "github.com/Casper-dev/Casper-server/core/commands" - "github.com/Casper-dev/Casper-server/casper/casper_utils" - "errors" - "github.com/ethereum/go-ethereum/core/types" -) - -type CasperServerHandler struct { -} - -func NewCasperServerHandler() *CasperServerHandler { - return &CasperServerHandler{} -} - -func (serverHandler *CasperServerHandler) Ping(ctx context.Context) (int64, error) { - fmt.Println("Ping received") - return int64(time.Now().Unix()), nil -} - -func (serverHandler *CasperServerHandler) SendUploadQuery(ctx context.Context, hash string, ipAddr string, size int64) (status string, err error) { - fmt.Println(hash + " " + ipAddr); - ///TODO: We might want to reimplement this without runCommand - status, err = runCommand(ctx, []string{"files", "cp", "/ipfs/" + hash, "/"}) - //status, err = runCommand(ctx, []string{"cat", "/ipfs/" + hash}) - if err == nil { - fmt.Println("no error"); - } - fmt.Println("Running events") - //intSize, err := strconv.ParseInt(size, 10, 64) - fmt.Println(err) - - fmt.Println("Waiting to get SC") - casper, client, auth := Casper_SC.GetSC() - - ///TODO: check actual size from network - - confirmUploadClosure := func() (tx *types.Transaction, err error) { - return casper.ConfirmUpload(auth, casper_utils.GetCasperNodeID(), hash, big.NewInt(size)) - } - if err != nil { - fmt.Println(err) - } - fmt.Println("Got SC") - Casper_SC.ValidateMineTX(confirmUploadClosure, client, auth) - - return -} - -func (serverHandler *CasperServerHandler) SendDownloadQuery(ctx context.Context, hash string, ipAddr string, wallet string) (status string, err error) { - fmt.Println(hash + " " + ipAddr) - decision.AllowedHashes[hash] = true - decision.Wallet = wallet - ///TODO: reimplement this in a more sane way - ///as it is now, this impl cannot serve more than one connection - fmt.Println("got download request") - return -} - -func (serverHandler *CasperServerHandler) SendDeleteQuery(ctx context.Context, hash string) (status string, err error) { - fmt.Println(hash) - ///TODO: We might want to reimplement this without runCommand - runCommand(ctx, []string{"ls", hash}) - status, err = runCommand(ctx, []string{"pin", "rm", hash}) - status, err = runCommand(ctx, []string{"files", "rm", "/" + hash}) - status, err = runCommand(ctx, []string{"block", "rm", hash}) - - casper, client, auth := Casper_SC.GetSC() - size := int64(commands.SizeOut) - - notifySpaceFreedClosure:= func() (tx *types.Transaction, err error) { - return casper.NotifySpaceFreed(auth, casper_utils.GetCasperNodeID(), hash, big.NewInt(size)) - } - fmt.Println("Got SC") - Casper_SC.ValidateMineTX(notifySpaceFreedClosure, client, auth) - if err == nil { - fmt.Println("no error"); - } - return -} - -func (serverHandler *CasperServerHandler) SendReplicationQuery(ctx context.Context, hash string, ip string, size int64) (status string, err error) { - client, _, _ := Casper_SC.GetSC() - status = "" - if verified, _ := client.VerifyReplication(nil, ip); verified { - ///Copied as is from SendUploadQuery - ///We might want to use different logic here soz for now we'll just copy func body from Upload - status, err = runCommand(ctx, []string{"files", "cp", "/ipfs/" + hash, "/"}) - ///status, err = runCommand(ctx, []string{"cat", "/ipfs/" + hash}) - if err == nil { - fmt.Println("no error"); - } - fmt.Println("Running events") - //intSize, err := strconv.ParseInt(size, 10, 64) - fmt.Println(err) - - fmt.Println("Waiting to get SC") - casper, client, auth := Casper_SC.GetSC() - - ///TODO: check actual size from network - ///tx, err := casper.ConfirmUpload(auth, casper_utils.GetCasperNodeID(), hash, big.NewInt(size)) - - fmt.Println("Got SC") - confirmUploadClosure := func() (tx *types.Transaction, err error) { - return casper.ConfirmUpload(auth, casper_utils.GetCasperNodeID(), hash, big.NewInt(size)) - } - Casper_SC.ValidateMineTX(confirmUploadClosure, client, auth) - } else { - err = errors.New("replication verification failed") - } - return -} - -func (serverHandler *CasperServerHandler) SendUpdateQuery(ctx context.Context, uuid string, hash string, size int64) (status string, err error) { - return -} - -func runCommand(ctx context.Context, args []string) (status string, err error) { - var invoc cmdInvocation - defer invoc.close() - - // parse the commandline into a command invocation - parseErr := invoc.Parse(ctx, args) - - // ok now handle parse error (which means cli input was wrong, - // e.g. incorrect number of args, or nonexistent subcommand) - if parseErr != nil { - fmt.Println(parseErr) - - // this was a user error, print help. - if invoc.cmd != nil { - // we need a newline space. - fmt.Fprintf(os.Stderr, "\n") - //printHelp(false, os.Stderr) - } - return "smells like ebola", parseErr - } - - // ok, finally, run the command invocation. - intrh, ctx := invoc.SetupInterruptHandler(ctx) - defer intrh.Close() - output, err := invoc.Run(ctx) - if err != nil { - fmt.Println(err) - return "smells like ebola", err - } - - // everything went better than expected :) - - _, err = io.Copy(os.Stdout, output) - if err != nil { - fmt.Println(err) - - // if this error was a client error, print short help too. - if isClientError(err) { - //printMetaHelp(os.Stderr) - } - return "smells like ebola", err - } - - return "Dis is da wei", nil -} diff --git a/cmd/ipfs_client/.gitignore b/cmd/ipfs_client/.gitignore new file mode 100644 index 0000000..ef5ec26 --- /dev/null +++ b/cmd/ipfs_client/.gitignore @@ -0,0 +1,3 @@ +ipfs_client +ipfs-test-cover +ipfs_client.exe diff --git a/cmd/ipfs_client/Rules.mk b/cmd/ipfs_client/Rules.mk new file mode 100644 index 0000000..175b098 --- /dev/null +++ b/cmd/ipfs_client/Rules.mk @@ -0,0 +1,50 @@ +include mk/header.mk +IPFS_BIN_$(d) := $(call go-curr-pkg-tgt) + +TGT_BIN += $(IPFS_BIN_$(d)) +TEST_GO_BUILD += $(d)-try-build +CLEAN += $(IPFS_BIN_$(d)) + +PATH := $(realpath $(d)):$(PATH) + +# disabled for now +# depend on *.pb.go files in the repo as Order Only (as they shouldn't be rebuilt if exist) +# DPES_OO_$(d) := diagnostics/pb/diagnostics.pb.go exchange/bitswap/message/pb/message.pb.go +# DEPS_OO_$(d) += merkledag/pb/merkledag.pb.go namesys/pb/namesys.pb.go +# DEPS_OO_$(d) += pin/internal/pb/header.pb.go unixfs/pb/unixfs.pb.go + +$(d)_flags =-ldflags="-X "github.com/ipfs/go-ipfs/repo/config".CurrentCommit=$(shell git rev-parse --short HEAD)" + +$(d)-try-build $(IPFS_BIN_$(d)): GOFLAGS += $(cmd/ipfs_flags) + +# uses second expansion to collect all $(DEPS_GO) +$(IPFS_BIN_$(d)): $(d) $$(DEPS_GO) ALWAYS #| $(DEPS_OO_$(d)) + $(go-build) + +TRY_BUILD_$(d)=$(addprefix $(d)-try-build-,$(SUPPORTED_PLATFORMS)) +$(d)-try-build: $(TRY_BUILD_$(d)) +.PHONY: $(d)-try-build + +$(TRY_BUILD_$(d)): PLATFORM = $(subst -, ,$(patsubst $<-try-build-%,%,$@)) +$(TRY_BUILD_$(d)): GOOS = $(word 1,$(PLATFORM)) +$(TRY_BUILD_$(d)): GOARCH = $(word 2,$(PLATFORM)) +$(TRY_BUILD_$(d)): $(d) $$(DEPS_GO) ALWAYS + GOOS=$(GOOS) GOARCH=$(GOARCH) $(go-try-build) +.PHONY: $(TRY_BUILD_$(d)) + +$(d)-install: GOFLAGS += $(cmd/ipfs_flags) +$(d)-install: $(d) $$(DEPS_GO) ALWAYS + $(GOCC) install $(go-flags-with-tags) ./cmd/ipfs +.PHONY: $(d)-install + +COVER_BIN_$(d) := $(d)/ipfs-test-cover +CLEAN += $(COVER_BIN_$(d)) + +$(COVER_BIN_$(d)): GOTAGS += testrunmain +$(COVER_BIN_$(d)): $(d) $$(DEPS_GO) ALWAYS + $(eval TMP_PKGS := $(shell $(GOCC) list -f '{{range .Deps}}{{.}} {{end}}' $(go-flags-with-tags) ./cmd/ipfs | sed 's/ /\n/g' | grep ipfs/go-ipfs | grep -v ipfs/go-ipfs/Godeps) $(call go-pkg-name,$<)) + $(eval TMP_LIST := $(call join-with,$(comma),$(TMP_PKGS))) + @echo $(GOCC) test $@ -c -covermode atomic -coverpkg ... $(go-flags-with-tags) ./$(@D) # for info + @$(GOCC) test -o $@ -c -covermode atomic -coverpkg $(TMP_LIST) $(go-flags-with-tags) ./$(@D) 2>&1 | (grep -v 'warning: no packages being tested' || true) + +include mk/footer.mk diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs_client/daemon.go similarity index 94% rename from cmd/ipfs/daemon.go rename to cmd/ipfs_client/daemon.go index 3b022a6..6b01515 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs_client/daemon.go @@ -11,23 +11,21 @@ import ( "sort" "sync" - cmds "github.com/Casper-dev/Casper-server/commands" - "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/core/commands" - "github.com/Casper-dev/Casper-server/core/corehttp" - "github.com/Casper-dev/Casper-server/core/corerepo" - nodeMount "github.com/Casper-dev/Casper-server/fuse/node" - "github.com/Casper-dev/Casper-server/repo/fsrepo" - migrate "github.com/Casper-dev/Casper-server/repo/fsrepo/migrations" + utilmain "gitlab.com/casperDev/Casper-server/cmd/ipfs_client/util" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core" + commands "gitlab.com/casperDev/Casper-server/core/commands" + corehttp "gitlab.com/casperDev/Casper-server/core/corehttp" + corerepo "gitlab.com/casperDev/Casper-server/core/corerepo" + nodeMount "gitlab.com/casperDev/Casper-server/fuse/node" + fsrepo "gitlab.com/casperDev/Casper-server/repo/fsrepo" + migrate "gitlab.com/casperDev/Casper-server/repo/fsrepo/migrations" mprome "gx/ipfs/QmSk46nSD78YiuNojYMS8NW6hSCjH95JajqqzzoychZgef/go-metrics-prometheus" "gx/ipfs/QmX3QZ5jHEPidwUrymXV1iSCSUhdGxj15sm2gP4jKMef7B/client_golang/prometheus" "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" iconn "gx/ipfs/QmfQAY7YU4fQi3sjGLs1hwkM2Aq7dxgDyoMjaKN4WBWvcB/go-libp2p-interface-conn" - - "github.com/Casper-dev/Casper-server/casper/casper_utils" - ) const ( @@ -48,6 +46,7 @@ const ( unrestrictedApiAccessKwd = "unrestricted-api" writableKwd = "writable" enableFloodSubKwd = "enable-pubsub-experiment" + enableIPNSPubSubKwd = "enable-namesys-pubsub" enableMultiplexKwd = "enable-mplex-experiment" // apiAddrKwd = "address-api" // swarmAddrKwd = "address-swarm" @@ -146,20 +145,22 @@ Headers. }, Options: []cmds.Option{ - cmds.BoolOption(initOptionKwd, "Initialize ipfs with default settings if not already initialized").Default(false), + cmds.BoolOption(initOptionKwd, "Initialize ipfs with default settings if not already initialized"), cmds.StringOption(routingOptionKwd, "Overrides the routing option").Default("dht"), - cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem").Default(false), - cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)").Default(false), + cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem"), + cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)"), cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."), cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."), - cmds.BoolOption(unrestrictedApiAccessKwd, "Allow API access to unlisted hashes").Default(false), - cmds.BoolOption(unencryptTransportKwd, "Disable transport encryption (for debugging protocols)").Default(false), - cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection").Default(false), + cmds.BoolOption(unrestrictedApiAccessKwd, "Allow API access to unlisted hashes"), + cmds.BoolOption(unencryptTransportKwd, "Disable transport encryption (for debugging protocols)"), + cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection"), cmds.BoolOption(adjustFDLimitKwd, "Check and raise file descriptor limits if needed").Default(true), - cmds.BoolOption(offlineKwd, "Run offline. Do not connect to the rest of the network but provide local API.").Default(false), + cmds.BoolOption(offlineKwd, "Run offline. Do not connect to the rest of the network but provide local API."), cmds.BoolOption(migrateKwd, "If true, assume yes at the migrate prompt. If false, assume no."), cmds.BoolOption(enableFloodSubKwd, "Instantiate the ipfs daemon with the experimental pubsub feature enabled."), + cmds.BoolOption(enableIPNSPubSubKwd, "Enable IPNS record distribution through pubsub; enables pubsub."), cmds.BoolOption(enableMultiplexKwd, "Add the experimental 'go-multiplex' stream muxer to libp2p on construction.").Default(true), + // TODO: add way to override addresses. tricky part: updating the config if also --init. // cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"), // cmds.StringOption(swarmAddrKwd, "Address for the swarm socket (overrides config)"), @@ -179,8 +180,6 @@ func defaultMux(path string) corehttp.ServeOption { } } -var fileDescriptorCheck = func() error { return nil } - func daemonFunc(req cmds.Request, res cmds.Response) { // Inject metrics before we do anything @@ -192,9 +191,8 @@ func daemonFunc(req cmds.Request, res cmds.Response) { // let the user know we're going. fmt.Printf("Initializing daemon...\n") - managefd, _, _ := req.Option(adjustFDLimitKwd).Bool() - if managefd { - if err := fileDescriptorCheck(); err != nil { + if managed, _, _ := req.Option(adjustFDLimitKwd).Bool(); managed { + if err := utilmain.ManageFdLimit(); err != nil { log.Errorf("setting file descriptor limit: %s", err) } } @@ -284,15 +282,17 @@ func daemonFunc(req cmds.Request, res cmds.Response) { offline, _, _ := req.Option(offlineKwd).Bool() pubsub, _, _ := req.Option(enableFloodSubKwd).Bool() + ipnsps, _, _ := req.Option(enableIPNSPubSubKwd).Bool() mplex, _, _ := req.Option(enableMultiplexKwd).Bool() // Start assembling node config ncfg := &core.BuildCfg{ Repo: repo, - Permament: true, // It is temporary way to signify that node is permament + Permament: true, // It is temporary way to signify that node is permanent Online: !offline, ExtraOpts: map[string]bool{ "pubsub": pubsub, + "ipnsps": ipnsps, "mplex": mplex, }, //TODO(Kubuxu): refactor Online vs Offline by adding Permanent vs Ephemeral @@ -303,8 +303,6 @@ func daemonFunc(req cmds.Request, res cmds.Response) { res.SetError(err, cmds.ErrNormal) return } - - fmt.Println(routingOption) switch routingOption { case routingOptionSupernodeKwd: res.SetError(errors.New("supernode routing was never fully implemented and has been removed"), cmds.ErrNormal) @@ -320,9 +318,6 @@ func daemonFunc(req cmds.Request, res cmds.Response) { return } - - ncfg.Routing = core.DHTClientOption - node, err := core.NewNode(req.Context(), ncfg) if err != nil { log.Error("error from node construction: ", err) @@ -338,9 +333,6 @@ func daemonFunc(req cmds.Request, res cmds.Response) { printSwarmAddrs(node) - go casper_utils.RegisterSC(node, cfg) - //go memory.ServeRPC() - defer func() { // We wait for the node to close first, as the node has children // that it will wait for before closing, such as the API server. @@ -412,6 +404,7 @@ func daemonFunc(req cmds.Request, res cmds.Response) { res.SetError(err, cmds.ErrNormal) } } + } // serveHTTPApi collects options, creates listener, prints status message and starts serving requests diff --git a/cmd/ipfs/dist/LICENSE b/cmd/ipfs_client/dist/LICENSE similarity index 100% rename from cmd/ipfs/dist/LICENSE rename to cmd/ipfs_client/dist/LICENSE diff --git a/cmd/ipfs/dist/README.md b/cmd/ipfs_client/dist/README.md similarity index 100% rename from cmd/ipfs/dist/README.md rename to cmd/ipfs_client/dist/README.md diff --git a/cmd/ipfs_client/dist/install.sh b/cmd/ipfs_client/dist/install.sh new file mode 100644 index 0000000..daf7185 --- /dev/null +++ b/cmd/ipfs_client/dist/install.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Installation script for ipfs. It tries to move $bin in one of the +# directories stored in $binpaths. + +INSTALL_DIR=$(dirname $0) + +bin="$INSTALL_DIR/ipfs" +binpaths="/usr/local/bin /usr/bin" + +# This variable contains a nonzero length string in case the script fails +# because of missing write permissions. +is_write_perm_missing="" + +for binpath in $binpaths; do + if mv "$bin" "$binpath/$bin" 2> /dev/null; then + echo "Moved $bin to $binpath" + exit 0 + else + if [ -d "$binpath" -a ! -w "$binpath" ]; then + is_write_perm_missing=1 + fi + fi +done + +echo "We cannot install $bin in one of the directories $binpaths" + +if [ -n "$is_write_perm_missing" ]; then + echo "It seems that we do not have the necessary write permissions." + echo "Perhaps try running this script as a privileged user:" + echo + echo " sudo $0" + echo +fi + +exit 1 diff --git a/cmd/ipfs/init.go b/cmd/ipfs_client/init.go similarity index 94% rename from cmd/ipfs/init.go rename to cmd/ipfs_client/init.go index f320575..afb2801 100644 --- a/cmd/ipfs/init.go +++ b/cmd/ipfs_client/init.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -9,13 +10,12 @@ import ( "path" "strings" - context "context" - assets "github.com/Casper-dev/Casper-server/assets" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - namesys "github.com/Casper-dev/Casper-server/namesys" - config "github.com/Casper-dev/Casper-server/repo/config" - fsrepo "github.com/Casper-dev/Casper-server/repo/fsrepo" + "gitlab.com/casperDev/Casper-server/assets" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/namesys" + "gitlab.com/casperDev/Casper-server/repo/config" + "gitlab.com/casperDev/Casper-server/repo/fsrepo" ) const ( @@ -49,7 +49,7 @@ environment variable: }, Options: []cmds.Option{ cmds.IntOption("bits", "b", "Number of bits to use in the generated RSA private key.").Default(nBitsForKeypairDefault), - cmds.BoolOption("empty-repo", "e", "Don't add and pin help files to the local storage.").Default(false), + cmds.BoolOption("empty-repo", "e", "Don't add and pin help files to the local storage."), cmds.StringOption("profile", "p", "Apply profile settings to config. Multiple profiles can be separated by ','"), // TODO need to decide whether to expose the override as a file or a @@ -73,6 +73,9 @@ environment variable: return nil }, Run: func(req cmds.Request, res cmds.Response) { + // needs to be called at least once + res.SetOutput(nil) + if req.InvocContext().Online { res.SetError(errors.New("init must be run offline only!"), cmds.ErrNormal) return diff --git a/cmd/ipfs_client/ipfs.go b/cmd/ipfs_client/ipfs.go new file mode 100644 index 0000000..718c50c --- /dev/null +++ b/cmd/ipfs_client/ipfs.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + + cmds "gitlab.com/casperDev/Casper-server/commands" + commands "gitlab.com/casperDev/Casper-server/core/commands" +) + +// This is the CLI root, used for executing commands accessible to CLI clients. +// Some subcommands (like 'ipfs daemon' or 'ipfs init') are only accessible here, +// and can't be called through the HTTP API. +var Root = &cmds.Command{ + Options: commands.Root.Options, + Helptext: commands.Root.Helptext, +} + +// commandsClientCmd is the "ipfs commands" command for local cli +var commandsClientCmd = commands.CommandsCmd(Root) + +// Commands in localCommands should always be run locally (even if daemon is running). +// They can override subcommands in commands.Root by defining a subcommand with the same name. +var localCommands = map[string]*cmds.Command{ + "daemon": daemonCmd, + "init": initCmd, + "commands": commandsClientCmd, +} +var localMap = make(map[*cmds.Command]bool) + +func init() { + // setting here instead of in literal to prevent initialization loop + // (some commands make references to Root) + Root.Subcommands = localCommands + + // copy all subcommands from commands.Root into this root (if they aren't already present) + for k, v := range commands.Root.Subcommands { + if _, found := Root.Subcommands[k]; !found { + Root.Subcommands[k] = v + } + } + + for _, v := range localCommands { + localMap[v] = true + } +} + +// NB: when necessary, properties are described using negatives in order to +// provide desirable defaults +type cmdDetails struct { + cannotRunOnClient bool + cannotRunOnDaemon bool + doesNotUseRepo bool + + // doesNotUseConfigAsInput describes commands that do not use the config as + // input. These commands either initialize the config or perform operations + // that don't require access to the config. + // + // pre-command hooks that require configs must not be run before these + // commands. + doesNotUseConfigAsInput bool + + // preemptsAutoUpdate describes commands that must be executed without the + // auto-update pre-command hook + preemptsAutoUpdate bool +} + +func (d *cmdDetails) String() string { + return fmt.Sprintf("on client? %t, on daemon? %t, uses repo? %t", + d.canRunOnClient(), d.canRunOnDaemon(), d.usesRepo()) +} + +func (d *cmdDetails) Loggable() map[string]interface{} { + return map[string]interface{}{ + "canRunOnClient": d.canRunOnClient(), + "canRunOnDaemon": d.canRunOnDaemon(), + "preemptsAutoUpdate": d.preemptsAutoUpdate, + "usesConfigAsInput": d.usesConfigAsInput(), + "usesRepo": d.usesRepo(), + } +} + +func (d *cmdDetails) usesConfigAsInput() bool { return !d.doesNotUseConfigAsInput } +func (d *cmdDetails) canRunOnClient() bool { return !d.cannotRunOnClient } +func (d *cmdDetails) canRunOnDaemon() bool { return !d.cannotRunOnDaemon } +func (d *cmdDetails) usesRepo() bool { return !d.doesNotUseRepo } + +// "What is this madness!?" you ask. Our commands have the unfortunate problem of +// not being able to run on all the same contexts. This map describes these +// properties so that other code can make decisions about whether to invoke a +// command or return an error to the user. +var cmdDetailsMap = map[string]cmdDetails{ + "init": {doesNotUseConfigAsInput: true, cannotRunOnDaemon: true, doesNotUseRepo: true}, + "daemon": {doesNotUseConfigAsInput: true, cannotRunOnDaemon: true}, + "commands": {doesNotUseRepo: true}, + "version": {doesNotUseConfigAsInput: true, doesNotUseRepo: true}, // must be permitted to run before init + "log": {cannotRunOnClient: true}, + "diag/cmds": {cannotRunOnClient: true}, + "repo/fsck": {cannotRunOnDaemon: true}, + "config/edit": {cannotRunOnDaemon: true, doesNotUseRepo: true}, +} diff --git a/cmd/ipfs_client/main.go b/cmd/ipfs_client/main.go new file mode 100644 index 0000000..da00e76 --- /dev/null +++ b/cmd/ipfs_client/main.go @@ -0,0 +1,733 @@ +// cmd/ipfs implements the primary CLI binary for ipfs +package main + +import ( + "context" + "errors" + "fmt" + "io" + "math/big" + "math/rand" + "net" + "net/url" + "os" + "os/signal" + "path/filepath" + "runtime/pprof" + "strings" + "sync" + "syscall" + "time" + + cmds "gitlab.com/casperDev/Casper-server/commands" + cli "gitlab.com/casperDev/Casper-server/commands/cli" + http "gitlab.com/casperDev/Casper-server/commands/http" + core "gitlab.com/casperDev/Casper-server/core" + coreCmds "gitlab.com/casperDev/Casper-server/core/commands" + "gitlab.com/casperDev/Casper-server/plugin/loader" + repo "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + fsrepo "gitlab.com/casperDev/Casper-server/repo/fsrepo" + + "gitlab.com/casperDev/Casper-SC/casper_sc" + + u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + loggables "gx/ipfs/QmT4PgCNdv73hnFAqzHqwW44q7M9PWpykSswHDxndquZbc/go-libp2p-loggables" + manet "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" + ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" + osh "gx/ipfs/QmXuBJ7DR6k3rmUEKtvVMhwjmXDuJgXXPUt4LQXKBMsU93/go-os-helper" + + "github.com/ethereum/go-ethereum/core/types" +) + +// log is the command logger +var log = logging.Logger("cmd/ipfs") + +var errRequestCanceled = errors.New("request canceled") + +const ( + EnvEnableProfiling = "IPFS_PROF" + cpuProfile = "ipfs.cpuprof" + heapProfile = "ipfs.memprof" +) + +type cmdInvocation struct { + path []string + cmd *cmds.Command + req cmds.Request + node *core.IpfsNode +} + +type exitErr int + +func (e exitErr) Error() string { + return fmt.Sprint("exit code", int(e)) +} + +// main roadmap: +// - parse the commandline to get a cmdInvocation +// - if user requests help, print it and exit. +// - run the command invocation +// - output the response +// - if anything fails, print error, maybe with help +func main() { + os.Exit(mainRet()) +} + +func connectToPeers() { + fmt.Println("Gettings peers") + caspersclient, client, auth, _ := Casper_SC.GetSC() + + prePayClosure := func() (*types.Transaction, error) { + return caspersclient.PrePay(auth, big.NewInt(int64(1337))) + } + Casper_SC.ValidateMineTX(prePayClosure, client, auth) + ///TODO: connect to bootstrap nodes +} + +func runCommand(ctx context.Context, args []string) (status string, err error) { + var invoc cmdInvocation + defer invoc.close() + + // parse the commandline into a command invocation + parseErr := invoc.Parse(ctx, args) + + // ok now handle parse error (which means cli input was wrong, + // e.g. incorrect number of args, or nonexistent subcommand) + if parseErr != nil { + fmt.Println(parseErr) + + // this was a user error, print help. + if invoc.cmd != nil { + // we need a newline space. + fmt.Fprintf(os.Stderr, "\n") + //printHelp(false, os.Stderr) + } + return "smells like ebola", parseErr + } + + // ok, finally, run the command invocation. + intrh, ctx := invoc.SetupInterruptHandler(ctx) + defer intrh.Close() + + _, err = invoc.Run(ctx) + if err != nil { + fmt.Println(err) + + // if this error was a client error, print short help too. + if isClientError(err) { + //printMetaHelp(os.Stderr) + } + return "smells like ebola", err + } + + return "Dis is da wei", nil +} + +func mainRet() int { + rand.Seed(time.Now().UnixNano()) + ctx := logging.ContextWithLoggable(context.Background(), loggables.Uuid("session")) + var err error + + // we'll call this local helper to output errors. + // this is so we control how to print errors in one place. + printErr := func(err error) { + fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) + } + + stopFunc, err := profileIfEnabled() + if err != nil { + printErr(err) + return 1 + } + defer stopFunc() // to be executed as late as possible + + var invoc cmdInvocation + defer invoc.close() + + // this is a local helper to print out help text. + // there's some considerations that this makes easier. + printHelp := func(long bool, w io.Writer) { + helpFunc := cli.ShortHelp + if long { + helpFunc = cli.LongHelp + } + + helpFunc("ipfs", Root, invoc.path, w) + } + + // this is a message to tell the user how to get the help text + printMetaHelp := func(w io.Writer) { + cmdPath := strings.Join(invoc.path, " ") + fmt.Fprintf(w, "Use 'ipfs %s --help' for information about this command\n", cmdPath) + } + + // Handle `ipfs help' + if len(os.Args) == 2 { + if os.Args[1] == "help" { + printHelp(false, os.Stdout) + return 0 + } else if os.Args[1] == "--version" { + os.Args[1] = "version" + } + } + + // parse the commandline into a command invocation + parseErr := invoc.Parse(ctx, os.Args[1:]) + + // BEFORE handling the parse error, if we have enough information + // AND the user requested help, print it out and exit + if invoc.req != nil { + longH, shortH, err := invoc.requestedHelp() + if err != nil { + printErr(err) + return 1 + } + if longH || shortH { + printHelp(longH, os.Stdout) + return 0 + } + } + + // ok now handle parse error (which means cli input was wrong, + // e.g. incorrect number of args, or nonexistent subcommand) + if parseErr != nil { + printErr(parseErr) + + // this was a user error, print help. + if invoc.cmd != nil { + // we need a newline space. + fmt.Fprintf(os.Stderr, "\n") + printHelp(false, os.Stderr) + } + return 1 + } + + // here we handle the cases where + // - commands with no Run func are invoked directly. + // - the main command is invoked. + if invoc.cmd == nil || invoc.cmd.Run == nil { + printHelp(false, os.Stdout) + return 0 + } + + // ok, finally, run the command invocation. + intrh, ctx := invoc.SetupInterruptHandler(ctx) + defer intrh.Close() + + output, err := invoc.Run(ctx) + if err != nil { + printErr(err) + + // if this error was a client error, print short help too. + if isClientError(err) { + printMetaHelp(os.Stderr) + } + return 1 + } + + // everything went better than expected :) + _, err = io.Copy(os.Stdout, output) + if err != nil { + printErr(err) + return 1 + } + return 0 +} + +func (i *cmdInvocation) Run(ctx context.Context) (output io.Reader, err error) { + + // check if user wants to debug. option OR env var. + debug, _, err := i.req.Option("debug").Bool() + if err != nil { + return nil, err + } + if debug || os.Getenv("IPFS_LOGGING") == "debug" { + u.Debug = true + logging.SetDebugLogging() + } + if u.GetenvBool("DEBUG") { + u.Debug = true + } + + res, err := callCommand(ctx, i.req, Root, i.cmd) + if err != nil { + return nil, err + } + + if err := res.Error(); err != nil { + return nil, err + } + + return res.Reader() +} + +func (i *cmdInvocation) constructNodeFunc(ctx context.Context) func() (*core.IpfsNode, error) { + return func() (n *core.IpfsNode, err error) { + if i.req == nil { + return nil, errors.New("constructing node without a request") + } + + cmdctx := i.req.InvocContext() + if cmdctx == nil { + return nil, errors.New("constructing node without a request context") + } + + r, err := fsrepo.Open(i.req.InvocContext().ConfigRoot) + if err != nil { // repo is owned by the node + return nil, err + } + + // ok everything is good. set it on the invocation (for ownership) + // and return it. + n, err = core.NewNode(ctx, &core.BuildCfg{ + Online: cmdctx.Online, + Repo: r, + }) + if err != nil { + return nil, err + } + n.SetLocal(true) + i.node = n + return i.node, nil + } +} + +func (i *cmdInvocation) close() { + // let's not forget teardown. If a node was initialized, we must close it. + // Note that this means the underlying req.Context().Node variable is exposed. + // this is gross, and should be changed when we extract out the exec Context. + if i.node != nil { + log.Info("Shutting down node...") + i.node.Close() + } +} + +func (i *cmdInvocation) Parse(ctx context.Context, args []string) error { + var err error + + i.req, i.cmd, i.path, err = cli.Parse(args, os.Stdin, Root) + if err != nil { + return err + } + + repoPath, err := getRepoPath(i.req) + if err != nil { + return err + } + log.Debugf("config path is %s", repoPath) + + // this sets up the function that will initialize the config lazily. + cmdctx := i.req.InvocContext() + cmdctx.ConfigRoot = repoPath + cmdctx.LoadConfig = loadConfig + // this sets up the function that will initialize the node + // this is so that we can construct the node lazily. + cmdctx.ConstructNode = i.constructNodeFunc(ctx) + + // if no encoding was specified by user, default to plaintext encoding + // (if command doesn't support plaintext, use JSON instead) + if !i.req.Option("encoding").Found() { + if i.req.Command().Marshalers != nil && i.req.Command().Marshalers[cmds.Text] != nil { + i.req.SetOption("encoding", cmds.Text) + } else { + i.req.SetOption("encoding", cmds.JSON) + } + } + + return nil +} + +func (i *cmdInvocation) requestedHelp() (short bool, long bool, err error) { + longHelp, _, err := i.req.Option("help").Bool() + if err != nil { + return false, false, err + } + shortHelp, _, err := i.req.Option("h").Bool() + if err != nil { + return false, false, err + } + return longHelp, shortHelp, nil +} + +func callPreCommandHooks(ctx context.Context, details cmdDetails, req cmds.Request, root *cmds.Command) error { + + log.Event(ctx, "callPreCommandHooks", &details) + log.Debug("calling pre-command hooks...") + + return nil +} + +func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd *cmds.Command) (cmds.Response, error) { + log.Info(config.EnvDir, " ", req.InvocContext().ConfigRoot) + + log.Infof("Setting request type to '%s'", cmds.CallerOptClient) + req.SetOption(cmds.CallerOpt, cmds.CallerOptClient) + + var res cmds.Response + + err := req.SetRootContext(ctx) + if err != nil { + return nil, err + } + + details, err := commandDetails(req.Path(), root) + if err != nil { + return nil, err + } + + client, err := commandShouldRunOnDaemon(*details, req, root) + if err != nil { + return nil, err + } + + err = callPreCommandHooks(ctx, *details, req, root) + if err != nil { + return nil, err + } + + if cmd.PreRun != nil { + err = cmd.PreRun(req) + if err != nil { + return nil, err + } + } + + if client != nil && !cmd.External { + log.Debug("executing command via API") + res, err = client.Send(req) + if err != nil { + if isConnRefused(err) { + err = repo.ErrApiNotRunning + } + + return nil, wrapContextCanceled(err) + } + + } else { + log.Debug("executing command locally") + + pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins") + if _, err := loader.LoadPlugins(pluginpath); err != nil { + return nil, err + } + + err := req.SetRootContext(ctx) + if err != nil { + return nil, err + } + + // Okay!!!!! NOW we can call the command. + res = root.Call(req) + + } + + if cmd.PostRun != nil { + cmd.PostRun(req, res) + } + + return res, nil +} + +// commandDetails returns a command's details for the command given by |path| +// within the |root| command tree. +// +// Returns an error if the command is not found in the Command tree. +func commandDetails(path []string, root *cmds.Command) (*cmdDetails, error) { + var details cmdDetails + // find the last command in path that has a cmdDetailsMap entry + cmd := root + for _, cmp := range path { + cmd = cmd.Subcommand(cmp) + if cmd == nil { + return nil, fmt.Errorf("subcommand %s should be in root", cmp) + } + + if cmdDetails, found := cmdDetailsMap[strings.Join(path, "/")]; found { + details = cmdDetails + } + } + return &details, nil +} + +// commandShouldRunOnDaemon determines, from commmand details, whether a +// command ought to be executed on an ipfs daemon. +// +// It returns a client if the command should be executed on a daemon and nil if +// it should be executed on a client. It returns an error if the command must +// NOT be executed on either. +func commandShouldRunOnDaemon(details cmdDetails, req cmds.Request, root *cmds.Command) (http.Client, error) { + path := req.Path() + // root command. + if len(path) < 1 { + return nil, nil + } + + if details.cannotRunOnClient && details.cannotRunOnDaemon { + return nil, fmt.Errorf("command disabled: %s", path[0]) + } + + if details.doesNotUseRepo && details.canRunOnClient() { + return nil, nil + } + + // at this point need to know whether api is running. we defer + // to this point so that we dont check unnecessarily + + // did user specify an api to use for this command? + apiAddrStr, _, err := req.Option(coreCmds.ApiOption).String() + if err != nil { + return nil, err + } + + client, err := getApiClient(req.InvocContext().ConfigRoot, apiAddrStr) + if err == repo.ErrApiNotRunning { + if apiAddrStr != "" && req.Command() != daemonCmd { + // if user SPECIFIED an api, and this cmd is not daemon + // we MUST use it. so error out. + return nil, err + } + + // ok for api not to be running + } else if err != nil { // some other api error + return nil, err + } + + if client != nil { + if details.cannotRunOnDaemon { + // check if daemon locked. legacy error text, for now. + log.Debugf("Command cannot run on daemon. Checking if daemon is locked") + if daemonLocked, _ := fsrepo.LockedByOtherProcess(req.InvocContext().ConfigRoot); daemonLocked { + return nil, cmds.ClientError("ipfs daemon is running. please stop it to run this command") + } + return nil, nil + } + + return client, nil + } + + if details.cannotRunOnClient { + return nil, cmds.ClientError("must run on the ipfs daemon") + } + + return nil, nil +} + +func isClientError(err error) bool { + if e, ok := err.(*cmds.Error); ok { + return e.Code == cmds.ErrClient + } + + return false +} + +func getRepoPath(req cmds.Request) (string, error) { + repoOpt, found, err := req.Option("config").String() + if err != nil { + return "", err + } + if found && repoOpt != "" { + return repoOpt, nil + } + + repoPath, err := fsrepo.BestKnownPath() + if err != nil { + return "", err + } + return repoPath, nil +} + +func loadConfig(path string) (*config.Config, error) { + return fsrepo.ConfigAt(path) +} + +// startProfiling begins CPU profiling and returns a `stop` function to be +// executed as late as possible. The stop function captures the memprofile. +func startProfiling() (func(), error) { + + // start CPU profiling as early as possible + ofi, err := os.Create(cpuProfile) + if err != nil { + return nil, err + } + pprof.StartCPUProfile(ofi) + go func() { + for range time.NewTicker(time.Second * 30).C { + err := writeHeapProfileToFile() + if err != nil { + log.Error(err) + } + } + }() + + stopProfiling := func() { + pprof.StopCPUProfile() + defer ofi.Close() // captured by the closure + } + return stopProfiling, nil +} + +func writeHeapProfileToFile() error { + mprof, err := os.Create(heapProfile) + if err != nil { + return err + } + defer mprof.Close() // _after_ writing the heap profile + return pprof.WriteHeapProfile(mprof) +} + +// IntrHandler helps set up an interrupt handler that can +// be cleanly shut down through the io.Closer interface. +type IntrHandler struct { + sig chan os.Signal + wg sync.WaitGroup +} + +func NewIntrHandler() *IntrHandler { + ih := &IntrHandler{} + ih.sig = make(chan os.Signal, 1) + return ih +} + +func (ih *IntrHandler) Close() error { + close(ih.sig) + ih.wg.Wait() + return nil +} + +// Handle starts handling the given signals, and will call the handler +// callback function each time a signal is catched. The function is passed +// the number of times the handler has been triggered in total, as +// well as the handler itself, so that the handling logic can use the +// handler's wait group to ensure clean shutdown when Close() is called. +func (ih *IntrHandler) Handle(handler func(count int, ih *IntrHandler), sigs ...os.Signal) { + signal.Notify(ih.sig, sigs...) + ih.wg.Add(1) + go func() { + defer ih.wg.Done() + count := 0 + for range ih.sig { + count++ + handler(count, ih) + } + signal.Stop(ih.sig) + }() +} + +func (i *cmdInvocation) SetupInterruptHandler(ctx context.Context) (io.Closer, context.Context) { + + intrh := NewIntrHandler() + ctx, cancelFunc := context.WithCancel(ctx) + + handlerFunc := func(count int, ih *IntrHandler) { + switch count { + case 1: + fmt.Println() // Prevent un-terminated ^C character in terminal + + ih.wg.Add(1) + go func() { + defer ih.wg.Done() + cancelFunc() + }() + + default: + fmt.Println("Received another interrupt before graceful shutdown, terminating...") + os.Exit(-1) + } + } + + intrh.Handle(handlerFunc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + + return intrh, ctx +} + +func profileIfEnabled() (func(), error) { + // FIXME this is a temporary hack so profiling of asynchronous operations + // works as intended. + if os.Getenv(EnvEnableProfiling) != "" { + stopProfilingFunc, err := startProfiling() // TODO maybe change this to its own option... profiling makes it slower. + if err != nil { + return nil, err + } + return stopProfilingFunc, nil + } + return func() {}, nil +} + +var apiFileErrorFmt string = `Failed to parse '%[1]s/api' file. + error: %[2]s +If you're sure go-ipfs isn't running, you can just delete it. +` +var checkIPFSUnixFmt = "Otherwise check:\n\tps aux | grep ipfs" +var checkIPFSWinFmt = "Otherwise check:\n\ttasklist | findstr ipfs" + +// getApiClient checks the repo, and the given options, checking for +// a running API service. if there is one, it returns a client. +// otherwise, it returns errApiNotRunning, or another error. +func getApiClient(repoPath, apiAddrStr string) (http.Client, error) { + var apiErrorFmt string + switch { + case osh.IsUnix(): + apiErrorFmt = apiFileErrorFmt + checkIPFSUnixFmt + case osh.IsWindows(): + apiErrorFmt = apiFileErrorFmt + checkIPFSWinFmt + default: + apiErrorFmt = apiFileErrorFmt + } + + var addr ma.Multiaddr + var err error + if len(apiAddrStr) != 0 { + addr, err = ma.NewMultiaddr(apiAddrStr) + if err != nil { + return nil, err + } + if len(addr.Protocols()) == 0 { + return nil, fmt.Errorf("mulitaddr doesn't provide any protocols") + } + } else { + addr, err = fsrepo.APIAddr(repoPath) + if err == repo.ErrApiNotRunning { + return nil, err + } + + if err != nil { + return nil, fmt.Errorf(apiErrorFmt, repoPath, err.Error()) + } + } + if len(addr.Protocols()) == 0 { + return nil, fmt.Errorf(apiErrorFmt, repoPath, "multiaddr doesn't provide any protocols") + } + return apiClientForAddr(addr) +} + +func apiClientForAddr(addr ma.Multiaddr) (http.Client, error) { + _, host, err := manet.DialArgs(addr) + if err != nil { + return nil, err + } + + return http.NewClient(host), nil +} + +func isConnRefused(err error) bool { + // unwrap url errors from http calls + if urlerr, ok := err.(*url.Error); ok { + err = urlerr.Err + } + + netoperr, ok := err.(*net.OpError) + if !ok { + return false + } + + return netoperr.Op == "dial" +} + +func wrapContextCanceled(err error) error { + if strings.Contains(err.Error(), "request canceled") { + err = errRequestCanceled + } + return err +} diff --git a/cmd/ipfs_client/main_test.go b/cmd/ipfs_client/main_test.go new file mode 100644 index 0000000..3922fdb --- /dev/null +++ b/cmd/ipfs_client/main_test.go @@ -0,0 +1,14 @@ +package main + +import ( + "testing" + + "gx/ipfs/QmQp2a2Hhb7F6eK2A5hN8f9aJy4mtkEikL9Zj4cgB7d1dD/go-ipfs-cmdkit" +) + +func TestIsCientErr(t *testing.T) { + t.Log("Only catch pointers") + if !isClientError(&cmdkit.Error{Code: cmdkit.ErrClient}) { + t.Errorf("misidentified error") + } +} diff --git a/cmd/ipfs/runmain_test.go b/cmd/ipfs_client/runmain_test.go similarity index 100% rename from cmd/ipfs/runmain_test.go rename to cmd/ipfs_client/runmain_test.go diff --git a/cmd/ipfs_client/util/ulimit.go b/cmd/ipfs_client/util/ulimit.go new file mode 100644 index 0000000..742a080 --- /dev/null +++ b/cmd/ipfs_client/util/ulimit.go @@ -0,0 +1,91 @@ +package util + +import ( + "errors" + "fmt" + "os" + "strconv" + "syscall" + + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" +) + +var log = logging.Logger("ulimit") + +var ( + supportsFDManagement = false + + // getlimit returns the soft and hard limits of file descriptors counts + getLimit func() (int64, int64, error) + // set limit sets the soft and hard limits of file descriptors counts + setLimit func(int64, int64) error +) + +// maxFds is the maximum number of file descriptors that go-ipfs +// can use. The default value is 1024. This can be overwritten by the +// IPFS_FD_MAX env variable +var maxFds = uint64(2048) + +// setMaxFds sets the maxFds value from IPFS_FD_MAX +// env variable if it's present on the system +func setMaxFds() { + // check if the IPFS_FD_MAX is set up and if it does + // not have a valid fds number notify the user + if val := os.Getenv("IPFS_FD_MAX"); val != "" { + + fds, err := strconv.ParseUint(val, 10, 64) + if err != nil { + log.Errorf("bad value for IPFS_FD_MAX: %s", err) + return + } + + maxFds = fds + } +} + +// ManageFdLimit raise the current max file descriptor count +// of the process based on the IPFS_FD_MAX value +func ManageFdLimit() error { + if !supportsFDManagement { + return nil + } + + setMaxFds() + soft, hard, err := getLimit() + if err != nil { + return err + } + + max := int64(maxFds) + + if max <= soft { + return nil + } + + // the soft limit is the value that the kernel enforces for the + // corresponding resource + // the hard limit acts as a ceiling for the soft limit + // an unprivileged process may only set it's soft limit to a + // alue in the range from 0 up to the hard limit + if err = setLimit(max, max); err != nil { + if err != syscall.EPERM { + return fmt.Errorf("error setting: ulimit: %s", err) + } + + // the process does not have permission so we should only + // set the soft value + if max > hard { + return errors.New( + "cannot set rlimit, IPFS_FD_MAX is larger than the hard limit", + ) + } + + if err = setLimit(max, hard); err != nil { + return fmt.Errorf("error setting ulimit wihout hard limit: %s", err) + } + } + + fmt.Printf("Successfully raised file descriptor limit to %d.\n", max) + + return nil +} diff --git a/cmd/ipfs_client/util/ulimit_freebsd.go b/cmd/ipfs_client/util/ulimit_freebsd.go new file mode 100644 index 0000000..d8b1d55 --- /dev/null +++ b/cmd/ipfs_client/util/ulimit_freebsd.go @@ -0,0 +1,27 @@ +// +build freebsd + +package util + +import ( + unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix" +) + +func init() { + supportsFDManagement = true + getLimit = freebsdGetLimit + setLimit = freebdsSetLimit +} + +func freebsdGetLimit() (int64, int64, error) { + rlimit := unix.Rlimit{} + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit) + return rlimit.Cur, rlimit.Max, err +} + +func freebdsSetLimit(soft int64, max int64) error { + rlimit := unix.Rlimit{ + Cur: soft, + Max: max, + } + return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit) +} diff --git a/cmd/ipfs_client/util/ulimit_test.go b/cmd/ipfs_client/util/ulimit_test.go new file mode 100644 index 0000000..ffba72e --- /dev/null +++ b/cmd/ipfs_client/util/ulimit_test.go @@ -0,0 +1,88 @@ +package util + +import ( + "fmt" + "os" + "strings" + "syscall" + "testing" +) + +func TestManageFdLimit(t *testing.T) { + t.Log("Testing file descriptor count") + if err := ManageFdLimit(); err != nil { + t.Errorf("Cannot manage file descriptors") + } + + if maxFds != uint64(2048) { + t.Errorf("Maximum file descriptors default value changed") + } +} + +func TestManageInvalidNFds(t *testing.T) { + t.Logf("Testing file descriptor invalidity") + var err error + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } + + rlimit := syscall.Rlimit{} + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + t.Fatal("Cannot get the file descriptor count") + } + + value := rlimit.Max + rlimit.Cur + if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil { + t.Fatal("Cannot set the IPFS_FD_MAX env variable") + } + + // call to check and set the maximum file descriptor from the env + setMaxFds() + + if err = ManageFdLimit(); err == nil { + t.Errorf("ManageFdLimit should return an error") + } else if err != nil { + flag := strings.Contains(err.Error(), + "cannot set rlimit, IPFS_FD_MAX is larger than the hard limit") + if !flag { + t.Errorf("ManageFdLimit returned unexpected error") + } + } + + // unset all previous operations + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } +} + +func TestManageFdLimitWithEnvSet(t *testing.T) { + t.Logf("Testing file descriptor manager with IPFS_FD_MAX set") + var err error + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } + + rlimit := syscall.Rlimit{} + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + t.Fatal("Cannot get the file descriptor count") + } + + value := rlimit.Max - rlimit.Cur + 1 + if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil { + t.Fatal("Cannot set the IPFS_FD_MAX env variable") + } + + setMaxFds() + if maxFds != value { + t.Errorf("The maxfds is not set from IPFS_FD_MAX") + } + + if err = ManageFdLimit(); err != nil { + t.Errorf("Cannot manage file descriptor count") + } + + // unset all previous operations + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } +} diff --git a/cmd/ipfs_client/util/ulimit_unix.go b/cmd/ipfs_client/util/ulimit_unix.go new file mode 100644 index 0000000..f931034 --- /dev/null +++ b/cmd/ipfs_client/util/ulimit_unix.go @@ -0,0 +1,27 @@ +// +build darwin linux netbsd openbsd + +package util + +import ( + unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix" +) + +func init() { + supportsFDManagement = true + getLimit = unixGetLimit + setLimit = unixSetLimit +} + +func unixGetLimit() (int64, int64, error) { + rlimit := unix.Rlimit{} + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit) + return int64(rlimit.Cur), int64(rlimit.Max), err +} + +func unixSetLimit(soft int64, max int64) error { + rlimit := unix.Rlimit{ + Cur: uint64(soft), + Max: uint64(max), + } + return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit) +} diff --git a/cmd/ipfs_client/util/ulimit_windows.go b/cmd/ipfs_client/util/ulimit_windows.go new file mode 100644 index 0000000..3cd9908 --- /dev/null +++ b/cmd/ipfs_client/util/ulimit_windows.go @@ -0,0 +1,7 @@ +// +build windows + +package util + +func init() { + supportsFDManagement = false +} diff --git a/cmd/ipfs_server/.gitignore b/cmd/ipfs_server/.gitignore new file mode 100644 index 0000000..110adf7 --- /dev/null +++ b/cmd/ipfs_server/.gitignore @@ -0,0 +1,3 @@ +ipfs_server +ipfs-test-cover +ipfs_server.exe diff --git a/cmd/ipfs/1.txt b/cmd/ipfs_server/1.txt similarity index 100% rename from cmd/ipfs/1.txt rename to cmd/ipfs_server/1.txt diff --git a/cmd/ipfs/Rules.mk b/cmd/ipfs_server/Rules.mk similarity index 100% rename from cmd/ipfs/Rules.mk rename to cmd/ipfs_server/Rules.mk diff --git a/cmd/ipfs_server/daemon.go b/cmd/ipfs_server/daemon.go new file mode 100644 index 0000000..7ee147a --- /dev/null +++ b/cmd/ipfs_server/daemon.go @@ -0,0 +1,748 @@ +package main + +import ( + "bufio" + "errors" + _ "expvar" + "fmt" + "net" + "net/http" + _ "net/http/pprof" + "os" + "sort" + "strings" + "sync" + + "gitlab.com/casperDev/Casper-server/casper/casper_utils" + + "gitlab.com/casperDev/Casper-server/casper/restapi" + + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core/commands" + "gitlab.com/casperDev/Casper-server/core/corehttp" + "gitlab.com/casperDev/Casper-server/core/corerepo" + nodeMount "gitlab.com/casperDev/Casper-server/fuse/node" + "gitlab.com/casperDev/Casper-server/repo/config" + "gitlab.com/casperDev/Casper-server/repo/fsrepo" + migrate "gitlab.com/casperDev/Casper-server/repo/fsrepo/migrations" + + mprome "gx/ipfs/QmSk46nSD78YiuNojYMS8NW6hSCjH95JajqqzzoychZgef/go-metrics-prometheus" + "gx/ipfs/QmX3QZ5jHEPidwUrymXV1iSCSUhdGxj15sm2gP4jKMef7B/client_golang/prometheus" + "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" + ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" + iconn "gx/ipfs/QmfQAY7YU4fQi3sjGLs1hwkM2Aq7dxgDyoMjaKN4WBWvcB/go-libp2p-interface-conn" + "gitlab.com/casperDev/Casper-server/casper/validation" +) + +const ( + adjustFDLimitKwd = "manage-fdlimit" + enableGCKwd = "enable-gc" + initOptionKwd = "init" + ipfsMountKwd = "mount-ipfs" + ipnsMountKwd = "mount-ipns" + migrateKwd = "migrate" + mountKwd = "mount" + offlineKwd = "offline" + routingOptionKwd = "routing" + routingOptionSupernodeKwd = "supernode" + routingOptionDHTClientKwd = "dhtclient" + routingOptionDHTKwd = "dht" + routingOptionNoneKwd = "none" + unencryptTransportKwd = "disable-transport-encryption" + unrestrictedApiAccessKwd = "unrestricted-api" + writableKwd = "writable" + enableFloodSubKwd = "enable-pubsub-experiment" + enableMultiplexKwd = "enable-mplex-experiment" + // apiAddrKwd = "address-api" + // swarmAddrKwd = "address-swarm" +) + +var daemonCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Run a network-connected IPFS node.", + ShortDescription: ` +'ipfs daemon' runs a persistent ipfs daemon that can serve commands +over the network. Most applications that use IPFS will do so by +communicating with a daemon over the HTTP API. While the daemon is +running, calls to 'ipfs' commands will be sent over the network to +the daemon. +`, + LongDescription: ` +The daemon will start listening on ports on the network, which are +documented in (and can be modified through) 'ipfs config Addresses'. +For example, to change the 'Gateway' port: + + ipfs config Addresses.Gateway /ip4/127.0.0.1/tcp/8082 + +The API address can be changed the same way: + + ipfs config Addresses.API /ip4/127.0.0.1/tcp/5002 + +Make sure to restart the daemon after changing addresses. + +By default, the gateway is only accessible locally. To expose it to +other computers in the network, use 0.0.0.0 as the ip address: + + ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080 + +Be careful if you expose the API. It is a security risk, as anyone could +control your node remotely. If you need to control the node remotely, +make sure to protect the port as you would other services or database +(firewall, authenticated proxy, etc). + +HTTP Headers + +ipfs supports passing arbitrary headers to the API and Gateway. You can +do this by setting headers on the API.HTTPHeaders and Gateway.HTTPHeaders +keys: + + ipfs config --json API.HTTPHeaders.X-Special-Header '["so special :)"]' + ipfs config --json Gateway.HTTPHeaders.X-Special-Header '["so special :)"]' + +Note that the value of the keys is an _array_ of strings. This is because +headers can have more than one value, and it is convenient to pass through +to other libraries. + +CORS Headers (for API) + +You can setup CORS headers the same way: + + ipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["example.com"]' + ipfs config --json API.HTTPHeaders.Access-Control-Allow-Methods '["PUT", "GET", "POST"]' + ipfs config --json API.HTTPHeaders.Access-Control-Allow-Credentials '["true"]' + +Shutdown + +To shutdown the daemon, send a SIGINT signal to it (e.g. by pressing 'Ctrl-C') +or send a SIGTERM signal to it (e.g. with 'kill'). It may take a while for the +daemon to shutdown gracefully, but it can be killed forcibly by sending a +second signal. + +IPFS_PATH environment variable + +ipfs uses a repository in the local file system. By default, the repo is +located at ~/.ipfs. To change the repo location, set the $IPFS_PATH +environment variable: + + export IPFS_PATH=/path/to/ipfsrepo + +Routing + +IPFS by default will use a DHT for content routing. There is a highly +experimental alternative that operates the DHT in a 'client only' mode that +can be enabled by running the daemon as: + + ipfs daemon --routing=dhtclient + +This will later be transitioned into a config option once it gets out of the +'experimental' stage. + +DEPRECATION NOTICE + +Previously, ipfs used an environment variable as seen below: + + export API_ORIGIN="http://localhost:8888/" + +This is deprecated. It is still honored in this version, but will be removed +in a future version, along with this notice. Please move to setting the HTTP +Headers. +`, + }, + + Options: []cmds.Option{ + cmds.BoolOption(initOptionKwd, "Initialize ipfs with default settings if not already initialized").Default(false), + cmds.StringOption(routingOptionKwd, "Overrides the routing option").Default("dht"), + cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem").Default(false), + cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)").Default(false), + cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."), + cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."), + cmds.BoolOption(unrestrictedApiAccessKwd, "Allow API access to unlisted hashes").Default(false), + cmds.BoolOption(unencryptTransportKwd, "Disable transport encryption (for debugging protocols)").Default(false), + cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection").Default(false), + cmds.BoolOption(adjustFDLimitKwd, "Check and raise file descriptor limits if needed").Default(true), + cmds.BoolOption(offlineKwd, "Run offline. Do not connect to the rest of the network but provide local API.").Default(false), + cmds.BoolOption(migrateKwd, "If true, assume yes at the migrate prompt. If false, assume no."), + cmds.BoolOption(enableFloodSubKwd, "Instantiate the ipfs daemon with the experimental pubsub feature enabled."), + cmds.BoolOption(enableMultiplexKwd, "Add the experimental 'go-multiplex' stream muxer to libp2p on construction.").Default(true), + // TODO: add way to override addresses. tricky part: updating the config if also --init. + // cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"), + // cmds.StringOption(swarmAddrKwd, "Address for the swarm socket (overrides config)"), + }, + Subcommands: map[string]*cmds.Command{}, + Run: daemonFunc, +} + +// defaultMux tells mux to serve path using the default muxer. This is +// mostly useful to hook up things that register in the default muxer, +// and don't provide a convenient http.Handler entry point, such as +// expvar and http/pprof. +func defaultMux(path string) corehttp.ServeOption { + return func(node *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + mux.Handle(path, http.DefaultServeMux) + return mux, nil + } +} + +var fileDescriptorCheck = func() error { return nil } + +func daemonFunc(req cmds.Request, res cmds.Response) { + // Inject metrics before we do anything + + err := mprome.Inject() + if err != nil { + log.Errorf("Injecting prometheus handler for metrics failed with message: %s\n", err.Error()) + } + + // let the user know we're going. + fmt.Printf("Initializing daemon...\n") + + managefd, _, _ := req.Option(adjustFDLimitKwd).Bool() + if managefd { + if err := fileDescriptorCheck(); err != nil { + log.Errorf("setting file descriptor limit: %s", err) + } + } + + ctx := req.InvocContext() + + go func() { + <-req.Context().Done() + fmt.Println("Received interrupt signal, shutting down...") + fmt.Println("(Hit ctrl-c again to force-shutdown the daemon.)") + }() + + // check transport encryption flag. + unencrypted, _, _ := req.Option(unencryptTransportKwd).Bool() + if unencrypted { + log.Warningf(`Running with --%s: All connections are UNENCRYPTED. + You will not be able to connect to regular encrypted networks.`, unencryptTransportKwd) + iconn.EncryptConnections = false + } + + // first, whether user has provided the initialization flag. we may be + // running in an uninitialized state. + initialize, _, err := req.Option(initOptionKwd).Bool() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + if initialize { + + cfg := ctx.ConfigRoot + if !fsrepo.IsInitialized(cfg) { + err := initWithDefaults(os.Stdout, cfg) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + } + } + + // acquire the repo lock _before_ constructing a node. we need to make + // sure we are permitted to access the resources (datastore, etc.) + repo, err := fsrepo.Open(ctx.ConfigRoot) + switch err { + default: + res.SetError(err, cmds.ErrNormal) + return + case fsrepo.ErrNeedMigration: + domigrate, found, _ := req.Option(migrateKwd).Bool() + fmt.Println("Found outdated fs-repo, migrations need to be run.") + + if !found { + domigrate = YesNoPrompt("Run migrations now? [y/N]") + } + + if !domigrate { + fmt.Println("Not running migrations of fs-repo now.") + fmt.Println("Please get fs-repo-migrations from https://dist.ipfs.io") + res.SetError(fmt.Errorf("fs-repo requires migration"), cmds.ErrNormal) + return + } + + err = migrate.RunMigration(fsrepo.RepoVersion) + if err != nil { + fmt.Println("The migrations of fs-repo failed:") + fmt.Printf(" %s\n", err) + fmt.Println("If you think this is a bug, please file an issue and include this whole log output.") + fmt.Println(" https://github.com/ipfs/fs-repo-migrations") + res.SetError(err, cmds.ErrNormal) + return + } + + repo, err = fsrepo.Open(ctx.ConfigRoot) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + case nil: + break + } + + if val, err := repo.GetConfigKey("Casper.TelegramAddress"); err != nil || val.(string) == config.DefaultCasperTelegramAddress { + fmt.Printf("Your current telegram address is not set. Please type it and press enter\n>") + buf := bufio.NewScanner(os.Stdin) + buf.Scan() + str := buf.Text() + println(str) + if err != nil { + println(err) + } + str = strings.TrimSpace(str) + err = repo.SetConfigKey("Casper.TelegramAddress", str) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + } + + cfg, err := ctx.GetConfig() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + if cfg.Casper.TelegramAddress == config.DefaultCasperTelegramAddress { + fmt.Printf("Your current telegram address is not set. Please type it and press enter\n> ") + scan := bufio.NewScanner(os.Stdin) + if scan.Scan() { + cfg.Casper.TelegramAddress = scan.Text() + err = repo.SetConfig(cfg) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + log.Debugf("new telegram name is '%s'", cfg.Casper.TelegramAddress) + } else { + res.SetError(fmt.Errorf("cannot get telegram address"), cmds.ErrNormal) + return + } + } + + offline, _, _ := req.Option(offlineKwd).Bool() + pubsub, _, _ := req.Option(enableFloodSubKwd).Bool() + mplex, _, _ := req.Option(enableMultiplexKwd).Bool() + + // Start assembling node config + ncfg := &core.BuildCfg{ + Repo: repo, + Permament: true, // It is temporary way to signify that node is permament + Online: !offline, + ExtraOpts: map[string]bool{ + "pubsub": pubsub, + "mplex": mplex, + }, + //TODO(Kubuxu): refactor Online vs Offline by adding Permanent vs Ephemeral + } + + routingOption, _, err := req.Option(routingOptionKwd).String() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + fmt.Println(routingOption) + switch routingOption { + case routingOptionSupernodeKwd: + res.SetError(errors.New("supernode routing was never fully implemented and has been removed"), cmds.ErrNormal) + return + case routingOptionDHTClientKwd: + ncfg.Routing = core.DHTClientOption + case routingOptionDHTKwd: + ncfg.Routing = core.DHTOption + case routingOptionNoneKwd: + ncfg.Routing = core.NilRouterOption + default: + res.SetError(fmt.Errorf("unrecognized routing option: %s", routingOption), cmds.ErrNormal) + return + } + + ncfg.Routing = core.DHTClientOption + + node, err := core.NewNode(req.Context(), ncfg) + if err != nil { + log.Error("error from node construction: ", err) + res.SetError(err, cmds.ErrNormal) + return + } + node.SetLocal(false) + + if node.PNetFingerpint != nil { + fmt.Println("Swarm is limited to private network of peers with the swarm key") + fmt.Printf("Swarm key fingerprint: %x\n", node.PNetFingerpint) + } + + printSwarmAddrs(node) + + var extAddrs []string + if cfg.Swarm.NAT.TraversalSC && len(cfg.Swarm.NAT.StunServers) > 0 { + log.Infof("NAT traversal is enabled and StunServers found in config. Trying to get external IP...") + laddr := net.JoinHostPort("0.0.0.0", cfg.Casper.ConnectionPort) + for _, stunIP := range cfg.Swarm.NAT.StunServers { + log.Debugf("Trying STUN-server: %s", stunIP) + if addr, err := connectStun(laddr, stunIP); err == nil { + log.Debugf("Received address: %s", addr.String()) + extAddrs = []string{addr.String()} + break + } else { + log.Error("Error while trying to get external IP:", err) + } + } + } + + go serveThrift(req.Context(), ctx) + go statusChecker() + go verificationWatcher(req.Context(), node) + go verificationRunner(req.Context()) + //go memory.ServeRPC() + + defer func() { + // We wait for the node to close first, as the node has children + // that it will wait for before closing, such as the API server. + node.Close() + + select { + case <-req.Context().Done(): + log.Info("Gracefully shut down daemon") + default: + } + }() + + err = casper_utils.RegisterSC(node, cfg, extAddrs...) + // TODO: make pings great again + pinger := &validation.Pinger{} + go pinger.RunPinger(req.Context()) + if err != nil { + res.SetError(fmt.Errorf("cant register SC: %v", err), cmds.ErrNormal) + return + } + + ctx.ConstructNode = func() (*core.IpfsNode, error) { + return node, nil + } + + // construct api endpoint - every time + err, apiErrc := serveHTTPApi(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + // construct fuse mountpoints - if the user provided the --mount flag + mount, _, err := req.Option(mountKwd).Bool() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + if mount && offline { + res.SetError(errors.New("mount is not currently supported in offline mode"), + cmds.ErrClient) + return + } + if mount { + if err := mountFuse(req); err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + } + + // repo blockstore GC - if --enable-gc flag is present + err, gcErrc := maybeRunGC(req, node) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + // construct http gateway - if it is set in the config + var gwErrc <-chan error + if len(cfg.Addresses.Gateway) > 0 { + var err error + err, gwErrc = serveHTTPGateway(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + } + + // initialize metrics collector + prometheus.MustRegister(&corehttp.IpfsNodeCollector{Node: node}) + + fmt.Printf("Daemon is ready\n") + // collect long-running errors and block for shutdown + // TODO(cryptix): our fuse currently doesnt follow this pattern for graceful shutdown + for err := range merge(apiErrc, gwErrc, gcErrc) { + if err != nil { + log.Error(err) + res.SetError(err, cmds.ErrNormal) + } + } +} + +// serveHTTPApi collects options, creates listener, prints status message and starts serving requests +func serveHTTPApi(req cmds.Request) (error, <-chan error) { + cfg, err := req.InvocContext().GetConfig() + if err != nil { + return fmt.Errorf("serveHTTPApi: GetConfig() failed: %s", err), nil + } + + apiAddr, _, err := req.Option(commands.ApiOption).String() + if err != nil { + return fmt.Errorf("serveHTTPApi: %s", err), nil + } + if apiAddr == "" { + apiAddr = cfg.Addresses.API + } + apiMaddr, err := ma.NewMultiaddr(apiAddr) + if err != nil { + return fmt.Errorf("serveHTTPApi: invalid API address: %q (err: %s)", apiAddr, err), nil + } + + apiLis, err := manet.Listen(apiMaddr) + if err != nil { + return fmt.Errorf("serveHTTPApi: manet.Listen(%s) failed: %s", apiMaddr, err), nil + } + // we might have listened to /tcp/0 - lets see what we are listing on + apiMaddr = apiLis.Multiaddr() + fmt.Printf("API server listening on %s\n", apiMaddr) + + // by default, we don't let you load arbitrary ipfs objects through the api, + // because this would open up the api to scripting vulnerabilities. + // only the webui objects are allowed. + // if you know what you're doing, go ahead and pass --unrestricted-api. + unrestricted, _, err := req.Option(unrestrictedApiAccessKwd).Bool() + if err != nil { + return fmt.Errorf("serveHTTPApi: Option(%s) failed: %s", unrestrictedApiAccessKwd, err), nil + } + gatewayOpt := corehttp.GatewayOption(false, corehttp.WebUIPaths...) + if unrestricted { + gatewayOpt = corehttp.GatewayOption(true, "/ipfs", "/ipns") + } + + var opts = []corehttp.ServeOption{ + corehttp.MetricsCollectionOption("api"), + corehttp.CommandsOption(*req.InvocContext()), + restapi.CasperOption(*req.InvocContext()), + restapi.CasperFileShareOption(), + corehttp.WebUIOption, + gatewayOpt, + corehttp.VersionOption(), + defaultMux("/debug/vars"), + defaultMux("/debug/pprof/"), + corehttp.MetricsScrapingOption("/debug/metrics/prometheus"), + corehttp.LogOption(), + } + + if len(cfg.Gateway.RootRedirect) > 0 { + opts = append(opts, corehttp.RedirectOption("", cfg.Gateway.RootRedirect)) + } + + node, err := req.InvocContext().ConstructNode() + if err != nil { + return fmt.Errorf("serveHTTPApi: ConstructNode() failed: %s", err), nil + } + + if err := node.Repo.SetAPIAddr(apiMaddr); err != nil { + return fmt.Errorf("serveHTTPApi: SetAPIAddr() failed: %s", err), nil + } + + errc := make(chan error) + go func() { + errc <- corehttp.Serve(node, apiLis.NetListener(), opts...) + close(errc) + }() + return nil, errc +} + +// printSwarmAddrs prints the addresses of the host +func printSwarmAddrs(node *core.IpfsNode) { + if !node.OnlineMode() { + fmt.Println("Swarm not listening, running in offline mode.") + return + } + + var lisAddrs []string + ifaceAddrs, err := node.PeerHost.Network().InterfaceListenAddresses() + if err != nil { + log.Errorf("failed to read listening addresses: %s", err) + } + for _, addr := range ifaceAddrs { + lisAddrs = append(lisAddrs, addr.String()) + } + sort.Sort(sort.StringSlice(lisAddrs)) + for _, addr := range lisAddrs { + fmt.Printf("Swarm listening on %s\n", addr) + } + + var addrs []string + for _, addr := range node.PeerHost.Addrs() { + addrs = append(addrs, addr.String()) + } + sort.Sort(sort.StringSlice(addrs)) + for _, addr := range addrs { + fmt.Printf("Swarm announcing %s\n", addr) + } + +} + +// serveHTTPGateway collects options, creates listener, prints status message and starts serving requests +func serveHTTPGateway(req cmds.Request) (error, <-chan error) { + cfg, err := req.InvocContext().GetConfig() + if err != nil { + return fmt.Errorf("serveHTTPGateway: GetConfig() failed: %s", err), nil + } + + gatewayMaddr, err := ma.NewMultiaddr(cfg.Addresses.Gateway) + if err != nil { + return fmt.Errorf("serveHTTPGateway: invalid gateway address: %q (err: %s)", cfg.Addresses.Gateway, err), nil + } + + writable, writableOptionFound, err := req.Option(writableKwd).Bool() + if err != nil { + return fmt.Errorf("serveHTTPGateway: req.Option(%s) failed: %s", writableKwd, err), nil + } + if !writableOptionFound { + writable = cfg.Gateway.Writable + } + + gwLis, err := manet.Listen(gatewayMaddr) + if err != nil { + return fmt.Errorf("serveHTTPGateway: manet.Listen(%s) failed: %s", gatewayMaddr, err), nil + } + // we might have listened to /tcp/0 - lets see what we are listing on + gatewayMaddr = gwLis.Multiaddr() + + if writable { + fmt.Printf("Gateway (writable) server listening on %s\n", gatewayMaddr) + } else { + fmt.Printf("Gateway (readonly) server listening on %s\n", gatewayMaddr) + } + + var opts = []corehttp.ServeOption{ + corehttp.MetricsCollectionOption("gateway"), + corehttp.CommandsROOption(*req.InvocContext()), + corehttp.VersionOption(), + corehttp.IPNSHostnameOption(), + corehttp.GatewayOption(writable, "/ipfs", "/ipns"), + } + + if len(cfg.Gateway.RootRedirect) > 0 { + opts = append(opts, corehttp.RedirectOption("", cfg.Gateway.RootRedirect)) + } + + node, err := req.InvocContext().ConstructNode() + if err != nil { + return fmt.Errorf("serveHTTPGateway: ConstructNode() failed: %s", err), nil + } + + errc := make(chan error) + go func() { + errc <- corehttp.Serve(node, gwLis.NetListener(), opts...) + close(errc) + }() + return nil, errc +} + +//collects options and opens the fuse mountpoint +func mountFuse(req cmds.Request) error { + cfg, err := req.InvocContext().GetConfig() + if err != nil { + return fmt.Errorf("mountFuse: GetConfig() failed: %s", err) + } + + fsdir, found, err := req.Option(ipfsMountKwd).String() + if err != nil { + return fmt.Errorf("mountFuse: req.Option(%s) failed: %s", ipfsMountKwd, err) + } + if !found { + fsdir = cfg.Mounts.IPFS + } + + nsdir, found, err := req.Option(ipnsMountKwd).String() + if err != nil { + return fmt.Errorf("mountFuse: req.Option(%s) failed: %s", ipnsMountKwd, err) + } + if !found { + nsdir = cfg.Mounts.IPNS + } + + node, err := req.InvocContext().ConstructNode() + if err != nil { + return fmt.Errorf("mountFuse: ConstructNode() failed: %s", err) + } + + err = nodeMount.Mount(node, fsdir, nsdir) + if err != nil { + return err + } + fmt.Printf("IPFS mounted at: %s\n", fsdir) + fmt.Printf("IPNS mounted at: %s\n", nsdir) + return nil +} + +func maybeRunGC(req cmds.Request, node *core.IpfsNode) (error, <-chan error) { + enableGC, _, err := req.Option(enableGCKwd).Bool() + if err != nil { + return err, nil + } + if !enableGC { + return nil, nil + } + + errc := make(chan error) + go func() { + errc <- corerepo.PeriodicGC(req.Context(), node) + close(errc) + }() + return nil, errc +} + +// merge does fan-in of multiple read-only error channels +// taken from http://blog.golang.org/pipelines +func merge(cs ...<-chan error) <-chan error { + var wg sync.WaitGroup + out := make(chan error) + + // Start an output goroutine for each input channel in cs. output + // copies values from c to out until c is closed, then calls wg.Done. + output := func(c <-chan error) { + for n := range c { + out <- n + } + wg.Done() + } + for _, c := range cs { + if c != nil { + wg.Add(1) + go output(c) + } + } + + // Start a goroutine to close out once all the output goroutines are + // done. This must start after the wg.Add call. + go func() { + wg.Wait() + close(out) + }() + return out +} + +func YesNoPrompt(prompt string) bool { + var s string + for i := 0; i < 3; i++ { + fmt.Printf("%s ", prompt) + fmt.Scanf("%s", &s) + switch s { + case "y", "Y": + return true + case "n", "N": + return false + case "": + return false + } + fmt.Println("Please press either 'y' or 'n'") + } + + return false +} diff --git a/cmd/ipfs_server/dist/LICENSE b/cmd/ipfs_server/dist/LICENSE new file mode 100644 index 0000000..c7386b3 --- /dev/null +++ b/cmd/ipfs_server/dist/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Juan Batiz-Benet + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cmd/ipfs_server/dist/README.md b/cmd/ipfs_server/dist/README.md new file mode 100644 index 0000000..8504e15 --- /dev/null +++ b/cmd/ipfs_server/dist/README.md @@ -0,0 +1,28 @@ +# ipfs commandline tool + +This is the [ipfs](http://ipfs.io) commandline tool. It contains a full ipfs node. + +## Install + +To install it, move the binary somewhere in your `$PATH`: + +```sh +sudo mv ipfs /usr/local/bin/ipfs +``` + +Or run `sudo install.sh` which does this for you. + +## Usage + +First, you must initialize your local ipfs node: + +```sh +ipfs init +``` + +This will give you directions to get started with ipfs. +You can always get help with: + +```sh +ipfs --help +``` diff --git a/cmd/ipfs/dist/install.sh b/cmd/ipfs_server/dist/install.sh similarity index 100% rename from cmd/ipfs/dist/install.sh rename to cmd/ipfs_server/dist/install.sh diff --git a/cmd/ipfs_server/init.go b/cmd/ipfs_server/init.go new file mode 100644 index 0000000..6fa2c81 --- /dev/null +++ b/cmd/ipfs_server/init.go @@ -0,0 +1,260 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "strings" + + "context" + + "gitlab.com/casperDev/Casper-server/assets" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/namesys" + "gitlab.com/casperDev/Casper-server/repo/config" + "gitlab.com/casperDev/Casper-server/repo/fsrepo" +) + +const ( + nBitsForKeypairDefault = 2048 +) + +var initCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Initializes ipfs config file.", + ShortDescription: ` +Initializes ipfs configuration files and generates a new keypair. + +If you are going to run IPFS in server environment, you may want to +initialize it using 'server' profile. + +Available profiles: + 'server' - Disables local host discovery, recommended when + running IPFS on machines with public IPv4 addresses. + 'test' - Reduces external interference of IPFS daemon, this + is useful when using the daemon in test environments. + +ipfs uses a repository in the local file system. By default, the repo is +located at ~/.ipfs. To change the repo location, set the $IPFS_PATH +environment variable: + + export IPFS_PATH=/path/to/ipfsrepo +`, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("default-config", false, false, "Initialize with the given configuration.").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.IntOption("bits", "b", "Number of bits to use in the generated RSA private key.").Default(nBitsForKeypairDefault), + cmds.BoolOption("empty-repo", "e", "Don't add and pin help files to the local storage.").Default(false), + cmds.StringOption("profile", "p", "Apply profile settings to config. Multiple profiles can be separated by ','").Default("nat,server"), + + // TODO need to decide whether to expose the override as a file or a + // directory. That is: should we allow the user to also specify the + // name of the file? + // TODO cmds.StringOption("event-logs", "l", "Location for machine-readable event logs."), + }, + PreRun: func(req cmds.Request) error { + daemonLocked, err := fsrepo.LockedByOtherProcess(req.InvocContext().ConfigRoot) + if err != nil { + return err + } + + log.Info("checking if daemon is running...") + if daemonLocked { + log.Debug("ipfs daemon is running") + e := "ipfs daemon is running. please stop it to run this command" + return cmds.ClientError(e) + } + + return nil + }, + Run: func(req cmds.Request, res cmds.Response) { + if req.InvocContext().Online { + res.SetError(errors.New("init must be run offline only!"), cmds.ErrNormal) + return + } + + empty, _, err := req.Option("e").Bool() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + nBitsForKeypair, _, err := req.Option("b").Int() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + var conf *config.Config + + f := req.Files() + if f != nil { + confFile, err := f.NextFile() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + conf = &config.Config{} + if err := json.NewDecoder(confFile).Decode(conf); err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + } + + profile, _, err := req.Option("profile").String() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + var profiles []string + if profile != "" { + profiles = strings.Split(profile, ",") + } + + if err := doInit(os.Stdout, req.InvocContext().ConfigRoot, empty, nBitsForKeypair, profiles, conf); err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + }, +} + +var errRepoExists = errors.New(`ipfs configuration file already exists! +Reinitializing would overwrite your keys. +`) + +func initWithDefaults(out io.Writer, repoRoot string) error { + return doInit(out, repoRoot, false, nBitsForKeypairDefault, []string{"nat", "server"}, nil) +} + +func doInit(out io.Writer, repoRoot string, empty bool, nBitsForKeypair int, confProfiles []string, conf *config.Config) error { + if _, err := fmt.Fprintf(out, "initializing IPFS node at %s\n", repoRoot); err != nil { + return err + } + + if err := checkWriteable(repoRoot); err != nil { + return err + } + + if fsrepo.IsInitialized(repoRoot) { + return errRepoExists + } + + if conf == nil { + var err error + conf, err = config.Init(out, nBitsForKeypair) + if err != nil { + return err + } + } + + for _, profile := range confProfiles { + transformer, ok := config.ConfigProfiles[profile] + if !ok { + return fmt.Errorf("invalid configuration profile: %s", profile) + } + + if err := transformer(conf); err != nil { + return err + } + } + + if err := fsrepo.Init(repoRoot, conf); err != nil { + return err + } + + if !empty { + if err := addDefaultAssets(out, repoRoot); err != nil { + return err + } + } + + return initializeIpnsKeyspace(repoRoot) +} + +func checkWriteable(dir string) error { + _, err := os.Stat(dir) + if err == nil { + // dir exists, make sure we can write to it + testfile := path.Join(dir, "test") + fi, err := os.Create(testfile) + if err != nil { + if os.IsPermission(err) { + return fmt.Errorf("%s is not writeable by the current user", dir) + } + return fmt.Errorf("unexpected error while checking writeablility of repo root: %s", err) + } + fi.Close() + return os.Remove(testfile) + } + + if os.IsNotExist(err) { + // dir doesnt exist, check that we can create it + return os.Mkdir(dir, 0775) + } + + if os.IsPermission(err) { + return fmt.Errorf("cannot write to %s, incorrect permissions", err) + } + + return err +} + +func addDefaultAssets(out io.Writer, repoRoot string) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + r, err := fsrepo.Open(repoRoot) + if err != nil { // NB: repo is owned by the node + return err + } + + nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r}) + if err != nil { + return err + } + defer nd.Close() + + dkey, err := assets.SeedInitDocs(nd) + if err != nil { + return fmt.Errorf("init: seeding init docs failed: %s", err) + } + log.Debugf("init: seeded init docs %s", dkey) + + if _, err = fmt.Fprintf(out, "to get started, enter:\n"); err != nil { + return err + } + + _, err = fmt.Fprintf(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey) + return err +} + +func initializeIpnsKeyspace(repoRoot string) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + r, err := fsrepo.Open(repoRoot) + if err != nil { // NB: repo is owned by the node + return err + } + + nd, err := core.NewNode(ctx, &core.BuildCfg{Repo: r}) + if err != nil { + return err + } + defer nd.Close() + + err = nd.SetupOfflineRouting() + if err != nil { + return err + } + + return namesys.InitializeKeyspace(ctx, nd.DAG, nd.Namesys, nd.Pinning, nd.PrivateKey) +} diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs_server/ipfs.go similarity index 97% rename from cmd/ipfs/ipfs.go rename to cmd/ipfs_server/ipfs.go index beef3a6..1808c37 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs_server/ipfs.go @@ -3,8 +3,8 @@ package main import ( "fmt" - cmds "github.com/Casper-dev/Casper-server/commands" - commands "github.com/Casper-dev/Casper-server/core/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" + commands "gitlab.com/casperDev/Casper-server/core/commands" ) // This is the CLI root, used for executing commands accessible to CLI clients. diff --git a/cmd/ipfs/main.go b/cmd/ipfs_server/main.go similarity index 95% rename from cmd/ipfs/main.go rename to cmd/ipfs_server/main.go index b0ffc34..d61dbd7 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs_server/main.go @@ -18,23 +18,23 @@ import ( "syscall" "time" - cmds "github.com/Casper-dev/Casper-server/commands" - cmdsCli "github.com/Casper-dev/Casper-server/commands/cli" - cmdsHttp "github.com/Casper-dev/Casper-server/commands/http" - core "github.com/Casper-dev/Casper-server/core" - coreCmds "github.com/Casper-dev/Casper-server/core/commands" - "github.com/Casper-dev/Casper-server/plugin/loader" - repo "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - fsrepo "github.com/Casper-dev/Casper-server/repo/fsrepo" + cmds "gitlab.com/casperDev/Casper-server/commands" + cmdsCli "gitlab.com/casperDev/Casper-server/commands/cli" + cmdsHttp "gitlab.com/casperDev/Casper-server/commands/http" + core "gitlab.com/casperDev/Casper-server/core" + coreCmds "gitlab.com/casperDev/Casper-server/core/commands" + "gitlab.com/casperDev/Casper-server/plugin/loader" + repo "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + fsrepo "gitlab.com/casperDev/Casper-server/repo/fsrepo" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" loggables "gx/ipfs/QmT4PgCNdv73hnFAqzHqwW44q7M9PWpykSswHDxndquZbc/go-libp2p-loggables" manet "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" osh "gx/ipfs/QmXuBJ7DR6k3rmUEKtvVMhwjmXDuJgXXPUt4LQXKBMsU93/go-os-helper" - logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" ) // log is the command logger @@ -65,6 +65,12 @@ func main() { os.Exit(mainRet()) } +// we'll call this helper to output errors. +// this is so we control how to print errors in one place. +func printErr(err error) { + fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) +} + func mainRet() int { rand.Seed(time.Now().UnixNano()) ctx := logging.ContextWithLoggable(context.Background(), loggables.Uuid("session")) @@ -72,12 +78,6 @@ func mainRet() int { var invoc cmdInvocation defer invoc.close() - // we'll call this local helper to output errors. - // this is so we control how to print errors in one place. - printErr := func(err error) { - fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) - } - stopFunc, err := profileIfEnabled() if err != nil { printErr(err) @@ -102,7 +102,6 @@ func mainRet() int { fmt.Fprintf(w, "Use 'ipfs %s --help' for information about this command\n", cmdPath) } - // Handle `ipfs help' if len(os.Args) == 2 { if os.Args[1] == "help" { printHelp(false, os.Stdout) @@ -143,11 +142,6 @@ func mainRet() int { return 1 } - // Start Thrift server only if it is daemon command - if invoc.cmd == daemonCmd { - go serve(invoc.req.InvocContext().ConfigRoot) - } - // here we handle the cases where // - commands with no Run func are invoked directly. // - the main command is invoked. @@ -305,6 +299,10 @@ func callPreCommandHooks(ctx context.Context, details cmdDetails, req cmds.Reque func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd *cmds.Command) (cmds.Response, error) { log.Info(config.EnvDir, " ", req.InvocContext().ConfigRoot) + + log.Infof("Setting request type to '%s'", cmds.CallerOptServer) + req.SetOption(cmds.CallerOpt, cmds.CallerOptServer) + var res cmds.Response err := req.SetRootContext(ctx) diff --git a/cmd/ipfs/main_test.go b/cmd/ipfs_server/main_test.go similarity index 86% rename from cmd/ipfs/main_test.go rename to cmd/ipfs_server/main_test.go index da23995..81bbf9e 100644 --- a/cmd/ipfs/main_test.go +++ b/cmd/ipfs_server/main_test.go @@ -3,7 +3,7 @@ package main import ( "testing" - "github.com/Casper-dev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/commands" ) func TestIsCientErr(t *testing.T) { diff --git a/cmd/ipfs_server/runmain_test.go b/cmd/ipfs_server/runmain_test.go new file mode 100644 index 0000000..8eecce3 --- /dev/null +++ b/cmd/ipfs_server/runmain_test.go @@ -0,0 +1,30 @@ +// +build testrunmain + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "testing" +) + +// this abuses go so much that I felt dirty writing this code +// but it is the only way to do it without writing custom compiler that would +// be a clone of go-build with go-test +func TestRunMain(t *testing.T) { + args := flag.Args() + os.Args = append([]string{os.Args[0]}, args...) + ret := mainRet() + + p := os.Getenv("IPFS_COVER_RET_FILE") + if len(p) != 0 { + ioutil.WriteFile(p, []byte(fmt.Sprintf("%d\n", ret)), 0777) + } + + // close outputs so go testing doesn't print anything + null, _ := os.Open(os.DevNull) + os.Stderr = null + os.Stdout = null +} diff --git a/cmd/ipfs/runwbootstrap.bat b/cmd/ipfs_server/runwbootstrap.bat similarity index 100% rename from cmd/ipfs/runwbootstrap.bat rename to cmd/ipfs_server/runwbootstrap.bat diff --git a/cmd/ipfs_server/serverdaemon.go b/cmd/ipfs_server/serverdaemon.go new file mode 100644 index 0000000..8c369c0 --- /dev/null +++ b/cmd/ipfs_server/serverdaemon.go @@ -0,0 +1,300 @@ +package main + +import ( + "context" + "fmt" + "math/big" + "math/rand" + "net" + "regexp" + "strings" + "time" + + cu "gitlab.com/casperDev/Casper-server/casper/casper_utils" + "gitlab.com/casperDev/Casper-server/casper/thrift" + "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/repo/fsrepo" + + "gitlab.com/casperDev/Casper-SC/casper" + "gitlab.com/casperDev/Casper-SC/casper_sc" + "gitlab.com/casperDev/Casper-server/casper/casper_utils" + + "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" + ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" + + "github.com/ethereum/go-ethereum/core/types" + reuse "github.com/libp2p/go-reuseport" + "github.com/willscott/goturn/client" + "github.com/fatih/color" + val "gitlab.com/casperDev/Casper-server/casper/validation" + "gitlab.com/casperDev/Casper-server/core" +) + +var pingTimeout = 30 * time.Second + +const ( + defaultStatusCheckInterval = 5 * time.Minute + defaultVerificationInitiationInterval = 60 * time.Minute +) + +func serveThrift(ctx context.Context, cctx *commands.Context) error { + log.Infof("Initializing thrift server...") + ///TODO: to use a custom port we need to find way for other providers to get that port + var thriftIP, thriftPort string + if cctx.ConfigRoot != "" { + if cfg, err := fsrepo.ConfigAt(cctx.ConfigRoot); err == nil && cfg != nil { + // TODO: Find out if API can be not IP4 or thrift can use IP6 + thriftIP, _ = ma.StringCast(cfg.Addresses.API).ValueForProtocol(ma.P_IP4) + thriftPort = cfg.Casper.ConnectionPort + } + } else { + log.Error("config is not provided") + } + + if thriftIP == "" { + thriftIP = cu.GetLocalIP() + thriftPort = "9090" + } + + thriftIP = "0.0.0.0" + + // always run server here + thriftAddr := net.JoinHostPort(thriftIP, thriftPort) + err := thrift.RunServerDefault(thriftAddr, NewCasperServerHandler(cctx.ConfigRoot)) + if err != nil { + log.Error("error running server:", err) + return err + } + + log.Infof("serving requests on %s ...", thriftAddr) + return nil +} + +func statusChecker() { + time.Sleep(10 * time.Second) ///TODO: wait for daemon to initialize + for ; ; { + casper, _, _, err := Casper_SC.GetSC() + if err != nil { + fmt.Println(err) + } + isBanned, err := casper.VerifyReplication(nil, casper_utils.GetLocalAddr().ID()) + fmt.Println() + if isBanned { + bgGreen := color.New(color.BgRed).PrintfFunc() + bgGreen(" Node is banned ") + fmt.Println() + } else { + bgGreen := color.New(color.BgGreen).PrintfFunc() + bgGreen(" Node is online ") + fmt.Println() + } + time.Sleep(defaultStatusCheckInterval) /// Node online check doesn't need to be frequent; we can even change it to subscription model + } +} + +var initiatedCheck string + +func verificationRunner(ctx context.Context) { + for ;; { + time.Sleep(defaultVerificationInitiationInterval) /// Node will run random uuid verification every defaultVerificationInitiationInterval(60 as of now) minutes + casperClient, _, auth, _ := Casper_SC.GetSC() + uuid := "lol" + casperClient.NotifyVerificationTarget(auth, uuid, casper_utils.GetLocalAddr().ID()) + } +} + +func verificationWatcher(ctx context.Context, node *core.IpfsNode) { + verificationTargetWatcher := func(ctx context.Context, node *core.IpfsNode) (func(log *casper.CasperVerificationTarget)) { + return func(log *casper.CasperVerificationTarget) { + if _, isStored := core.UUIDInfoCache.Load(log.UUID); isStored { + val.PerformValidation(ctx, node, log.UUID) + } + } + }(ctx, node) + casper_utils.SubscribeToVerificationTargetLogs(ctx, verificationTargetWatcher) + + verificationConsensusWatcher := func(ctx context.Context) (func(log *casper.CasperConsensusResult)) { + return func(log *casper.CasperConsensusResult) { + if _, isStored := core.UUIDInfoCache.Load(log.UUID); isStored { + for _, peer := range log.Consensus { + if string(peer[:31]) == casper_utils.GetLocalAddr().ID() { + repairFile(ctx, log.UUID) + } + } + } + } + }(ctx) + casper_utils.SubscribeToVerificationConsensusLogs(ctx, verificationConsensusWatcher) +} + +func repairFile(ctx context.Context, UUID string) { + fmt.Println("Repairing file", UUID) + /// Make file repair +} + +// TODO: implement timeout +const connectStunTimeout = 2 * time.Minute +const connectStunKeepAlive = time.Hour + +func connectStun(laddr, raddr string) (ma.Multiaddr, error) { + var d reuse.Dialer + if laddr != "" { + netladdr, err := reuse.ResolveAddr("tcp", laddr) + if err != nil { + return nil, err + } + d.D.LocalAddr = netladdr + } + d.D.Timeout = connectStunTimeout + d.D.KeepAlive = connectStunKeepAlive + + conn, err := d.Dial("tcp", raddr) + if err != nil { + log.Error("Could not connect to STUN server:", err) + return nil, err + } + + // We do not close connection intentionally, so that + // our external (outer) address remains the same + //defer conn.Close() + c := client.StunClient{Conn: conn} + address, err := c.Bind() + if err != nil { + log.Error("Failed to bind:", err) + return nil, err + } + + // StunClient.Bind() provides its own 'net.Addr' implementation + // which cannot be used as argument to FromNetAddr. + // So we construct net.TCPAddr here explicitly. + tcp, err := net.ResolveTCPAddr(address.Network(), address.String()) + if err != nil { + log.Error(err) + return nil, err + } + + return manet.FromNetAddr(tcp) +} + +func runPinger(ctx context.Context) { + ///beforeTimeout := time.Duration(60 - time.Now().Minute()) * time.Second + beforeTimeout := time.Duration(30) * time.Second + fmt.Printf("Waiting for %s\n", beforeTimeout) + time.Sleep(beforeTimeout) + + for { + casperclient, client, auth, _ := Casper_SC.GetSC() + fmt.Println("auth nonce", auth.Nonce) + nonce, _ := client.PendingNonceAt(ctx, auth.From) + fmt.Println("pending nonce", nonce) + //sink := make(chan *casper.CasperReturnString) + //_, err = casperclient.WatchReturnString(nil, sink) + nonce, _ = client.PendingNonceAt(ctx, auth.From) + fmt.Println("pending nonce", nonce) + + rer := regexp.MustCompile("(/ip4/[/\\d\\w.]+)") + ipRet := strings.TrimSpace(Casper_SC.ValidateMineTX(func() (*types.Transaction, error) { + return casperclient.GetPingTarget(auth, cu.GetLocalAddr().String()) + }, client, auth)) + fmt.Println("waiting for event") + //retString := <-sink + //fmt.Println("event ret ", retString.Val) + + re := regexp.MustCompile("/.+?/(.+?)/") + fmt.Println("got ip !!", ipRet, "!!") + ipRet = rer.FindString(ipRet) + fmt.Println("got ip !!", ipRet, "!!") + fmt.Println(len(ipRet)) + ips := re.FindStringSubmatch(ipRet) + if len(ips) >= 1 { + log.Debugf("Ping: %s", ips[0]) + if timestamp, err := callPing(ctx, net.JoinHostPort(ips[0], "9090")); err == nil { + log.Debugf("Ping timestamp: %d", timestamp) + success := timestamp >= time.Now().Unix()-int64(pingTimeout.Seconds()) + nonce, _ = client.PendingNonceAt(ctx, auth.From) + log.Debugf("Pending nonce", nonce) + resultData := Casper_SC.ValidateMineTX(func() (*types.Transaction, error) { + return casperclient.SendPingResult(auth, ipRet, success) + }, client, auth) + + ///TODO: change to working regex + re := regexp.MustCompile("Banned!") + fmt.Println("res data ", resultData) + if re.FindString(resultData) != "" { + fmt.Println("Go go replication~!") + go startReplication(ctx, ipRet, casperclient) + } + } + } + + ///pingTimeout := time.Duration(60-time.Now().Minute()+(rand.Int()%59)) * time.Second + pingWait := time.Duration(10+rand.Intn(20)) * time.Second + fmt.Printf("Sleeping for %s\n", pingWait) + time.Sleep(pingTimeout) + } +} + +func callPing(ctx context.Context, ip string) (timestamp int64, err error) { + defer func() { + if r := recover(); r != nil { + fmt.Println("Recovered in f", r) + } + }() + + ts, err := thrift.RunClientClosure(ip, func(client *thrift.ThriftClient) (interface{}, error) { + return client.Ping(ctx) + }) + if err != nil { + return 0, err + } + return ts.(int64), nil +} + +func startReplication(ctx context.Context, ip string, casperclient *casper.Casper) { + // We get number of banned provider files + n, err := casperclient.GetNumberOfFiles(nil, ip) + if err != nil { + log.Error(err) + return + } + + for i := int64(0); i < n.Int64(); i++ { + // For each file we get its hash and size + hash, size, err := casperclient.GetFile(nil, ip, big.NewInt(i)) + if err != nil { + log.Error(err) + continue + } + + log.Debug("hs", hash, size) + + // Search for a new peer to store the file + go replicateFile(ctx, hash, size, casperclient) + } +} + +func replicateFile(ctx context.Context, hash string, size *big.Int, casperclient *casper.Casper) (ret string) { + defer func() { + if re := recover(); re != nil { + fmt.Println(re) + } + }() + + tx, err := casperclient.GetPeers(nil, size) + if err != nil { + fmt.Println(err) + } + // GetPeers returns 4 ips, but we need only one of them + filteredIP, err := casperclient.GetIpPort(nil, tx.Id1) + if err != nil { + fmt.Println(err) + } + + if filteredIP != "" { + ip := net.JoinHostPort(filteredIP, "9090") + thrift.RunClientClosure(ip, func(client *thrift.ThriftClient) (interface{}, error) { + return client.SendReplicationQuery(ctx, hash, ip, size.Int64()) + }) + } + return +} diff --git a/cmd/ipfs_server/serverhandler.go b/cmd/ipfs_server/serverhandler.go new file mode 100644 index 0000000..57fc881 --- /dev/null +++ b/cmd/ipfs_server/serverhandler.go @@ -0,0 +1,260 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "io" + "math/big" + "net" + "os" + "time" + + "gitlab.com/casperDev/Casper-SC/casper_sc" + cu "gitlab.com/casperDev/Casper-server/casper/casper_utils" + uid "gitlab.com/casperDev/Casper-server/casper/uuid" + val "gitlab.com/casperDev/Casper-server/casper/validation" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core/commands" + "gitlab.com/casperDev/Casper-server/exchange/bitswap/decision" + "gitlab.com/casperDev/Casper-server/repo/fsrepo" + "gitlab.com/casperDev/Casper-thrift/casperproto" + + "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" + "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" + "gx/ipfs/QmeS8cCKawUwejVrsBtmC1toTXmwVWZGiRJqzgTURVWeF9/go-ipfs-addr" + + "github.com/ethereum/go-ethereum/core/types" +) + +type CasperServerHandler struct { + configRoot string +} + +func NewCasperServerHandler(configRoot string) *CasperServerHandler { + return &CasperServerHandler{configRoot: configRoot} +} + +func (_ *CasperServerHandler) NodeID() string { + return cu.GetLocalAddr().String() +} + +func (sh *CasperServerHandler) GetNode(ctx context.Context) (*core.IpfsNode, error) { + repo, err := fsrepo.Open(sh.configRoot) + if err != nil { + return nil, err + } + + return core.NewNode(ctx, &core.BuildCfg{Online: false, Repo: repo}) +} + +func (sh *CasperServerHandler) GetFileChecksum(ctx context.Context, uuid string, first, last int64, salt string) (string, error) { + log.Debugf("Thrift: GetFileChecksum(%s, %d, %d, %s)", uuid, first, last, salt) + + //id := uid.UUIDToCid(base58.Decode(uuid)) + mhash, err := multihash.FromB58String(uuid) + if err != nil { + return "", err + } + id := cid.NewCidV0(mhash) + + n, err := sh.GetNode(ctx) + if err != nil { + return "", err + } + + node, err := n.DAG.Get(ctx, id) + if err != nil { + return "", err + } + + cs, err := val.ChecksumSalt(ctx, node, first, last, n.DAG, []byte(salt)) + if err != nil { + return "", err + } + + return cs.B58String(), nil +} + +func (serverHandler *CasperServerHandler) Ping(ctx context.Context) (int64, error) { + log.Debugf("Thrift: Ping()") + return time.Now().Unix(), nil +} + +func (serverHandler *CasperServerHandler) SendUploadQuery(ctx context.Context, hash string, ipAddr string, size int64) (status string, err error) { + log.Debugf("Thrift: SendUploadQuery(%s, %s, %d)", hash, ipAddr, size) + + var ipList []string + err = json.Unmarshal([]byte(ipAddr), &ipList) + if err != nil { + return "", err + } + + log.Debugf("Received peers: %v", ipList) + + ///TODO: We might want to reimplement this without runCommand + status, err = runCommand(ctx, []string{"files", "cp", "/ipfs/" + hash, "/"}) + //status, err = runCommand(ctx, []string{"cat", "/ipfs/" + hash}) + if err != nil { + return "", err + } + + casper, client, auth, _ := Casper_SC.GetSC() + + ///TODO: check actual size from network + Casper_SC.ValidateMineTX(func() (tx *types.Transaction, err error) { + return casper.ConfirmUpload(auth, serverHandler.NodeID(), hash, big.NewInt(size)) + }, client, auth) + + return +} + +func (serverHandler *CasperServerHandler) SendDownloadQuery(ctx context.Context, hash string, ipAddr string, wallet string) (status string, err error) { + log.Debugf("Thrift: SendDownloadQuery(%s, %s, %s)", hash, ipAddr, wallet) + + decision.AllowHash(hash, wallet) + return "", nil +} + +func (serverHandler *CasperServerHandler) SendDeleteQuery(ctx context.Context, hash string) (status string, err error) { + log.Debugf("Thrift: SendDeleteQuery(%s)", hash) + + ///TODO: We might want to reimplement this without runCommand + status, err = runCommand(ctx, []string{"pin", "rm", hash}) + status, err = runCommand(ctx, []string{"files", "rm", "/" + hash}) + status, err = runCommand(ctx, []string{"block", "rm", hash}) + if err != nil { + return "", err + } + + casper, client, auth, _ := Casper_SC.GetSC() + Casper_SC.ValidateMineTX(func() (tx *types.Transaction, err error) { + return casper.NotifySpaceFreed(auth, serverHandler.NodeID(), hash, big.NewInt(int64(commands.SizeOut))) + }, client, auth) + + return "", nil +} + +func (serverHandler *CasperServerHandler) SendReplicationQuery(ctx context.Context, hash string, ip string, size int64) (status string, err error) { + log.Debugf("Thrift: SendReplicationQuery(%s, %s, %d)", hash, ip, size) + + client, _, _, _ := Casper_SC.GetSC() + verified, err := client.VerifyReplication(nil, ip) + if err != nil { + return "", err + } + if verified { + status, err = runCommand(ctx, []string{"files", "cp", "/ipfs/" + hash, "/"}) + if err != nil { + return + } + + ///TODO: check actual size from network + casper, client, auth, _ := Casper_SC.GetSC() + Casper_SC.ValidateMineTX(func() (tx *types.Transaction, err error) { + return casper.ConfirmUpload(auth, serverHandler.NodeID(), hash, big.NewInt(size)) + }, client, auth) + return status, nil + } + return "", errors.New("replication verification failed") +} + +func (serverHandler *CasperServerHandler) SendUpdateQuery(ctx context.Context, uuid string, hash string, size int64) (status string, err error) { + log.Debugf("Thrift: SendUpdateQuery(%s, %s, %d)", uuid, hash, size) + + status, err = runCommand(ctx, []string{"upd", uuid, hash}) + if err != nil { + return + } + + h := uid.UUIDToHash(base58.Decode(uuid)).B58String() + + casper, client, auth, _ := Casper_SC.GetSC() + Casper_SC.ValidateMineTX(func() (tx *types.Transaction, err error) { + return casper.ConfirmUpdate(auth, serverHandler.NodeID(), h, big.NewInt(size)) + }, client, auth) + + return +} + +// This is the func that every node invokes on CHECK INITIATOR other right after +// message with UUID has been appeared in SC logs +func (sh *CasperServerHandler) SendVerificationQuery(ctx context.Context, uuid string, ninfo *casperproto.NodeInfo) error { + log.Debugf("Thrift: VerificationQuery(%s, %+v)", uuid, ninfo) + + addr, err := ipfsaddr.ParseString(ninfo.IpfsAddr) + if err != nil { + log.Error(err) + return err + } + taddr, err := net.ResolveTCPAddr("tcp", ninfo.ThriftAddr) + if err != nil { + log.Error(err) + return err + } + + val.RegisterUUIDProvider(uuid, addr, taddr) + + return nil +} + +func (sh *CasperServerHandler) SendChunkInfo(ctx context.Context, cinfo *casperproto.ChunkInfo) error { + log.Debugf("Thrift: ChunkSectionInfo(%+v)", cinfo) + go val.CollectResultsAndRespond(context.Background(), cinfo, sh.configRoot) + log.Debugf("fihish ChunkSectionInfo()") + return nil +} + +func (sh *CasperServerHandler) SendChecksumHash(ctx context.Context, uuid string, ipfsAddr string, hashDiffuse string) error { + log.Debugf("Thrift: SendChecksumHash(%s, %s)", uuid, ipfsAddr) + addr, err := ipfsaddr.ParseString(ipfsAddr) + if err != nil { + return err + } + val.AddRound1Result(ctx, uuid, addr, hashDiffuse) + log.Debugf("finish SendChecksumHash()") + return nil +} + +func (serverHandler *CasperServerHandler) SendValidationResults(ctx context.Context, uuid string, ipfsAddr string, addrToHash map[string]string) error { + log.Debugf("Thrift: SendValidationResults(%s, %s)", uuid, ipfsAddr, addrToHash) + + // TODO Determine who is bad provider (if any) and send to SC + + return nil +} + +func runCommand(ctx context.Context, args []string) (status string, err error) { + var invoc cmdInvocation + defer invoc.close() + + // parse the commandline into a command invocation + parseErr := invoc.Parse(ctx, args) + + // ok now handle parse error (which means cli input was wrong, + // e.g. incorrect number of args, or nonexistent subcommand) + if parseErr != nil { + // panic(parseErr.Error()) + printErr(parseErr) + return "smells like ebola", parseErr + } + + // ok, finally, run the command invocation. + intrh, ctx := invoc.SetupInterruptHandler(ctx) + defer intrh.Close() + output, err := invoc.Run(ctx) + if err != nil { + printErr(err) + return "smells like ebola", err + } + + // everything went better than expected :) + _, err = io.Copy(os.Stdout, output) + if err != nil { + printErr(err) + return "smells like ebola", err + } + + return "Dis is da wei", nil +} diff --git a/cmd/ipfs/ulimit.go b/cmd/ipfs_server/ulimit.go similarity index 100% rename from cmd/ipfs/ulimit.go rename to cmd/ipfs_server/ulimit.go diff --git a/cmd/ipfs/ulimit_freebsd.go b/cmd/ipfs_server/ulimit_freebsd.go similarity index 100% rename from cmd/ipfs/ulimit_freebsd.go rename to cmd/ipfs_server/ulimit_freebsd.go diff --git a/cmd/ipfs/ulimit_unix.go b/cmd/ipfs_server/ulimit_unix.go similarity index 100% rename from cmd/ipfs/ulimit_unix.go rename to cmd/ipfs_server/ulimit_unix.go diff --git a/cmd/ipfswatch/ipfswatch_test.go b/cmd/ipfswatch/ipfswatch_test.go index 9abfa9f..093e533 100644 --- a/cmd/ipfswatch/ipfswatch_test.go +++ b/cmd/ipfswatch/ipfswatch_test.go @@ -3,7 +3,7 @@ package main import ( "testing" - "github.com/Casper-dev/Casper-server/thirdparty/assert" + "gitlab.com/casperDev/Casper-server/thirdparty/assert" ) func TestIsHidden(t *testing.T) { diff --git a/cmd/ipfswatch/main.go b/cmd/ipfswatch/main.go index eb3500e..e714041 100644 --- a/cmd/ipfswatch/main.go +++ b/cmd/ipfswatch/main.go @@ -9,14 +9,14 @@ import ( "path/filepath" "syscall" - commands "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - corehttp "github.com/Casper-dev/Casper-server/core/corehttp" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" - config "github.com/Casper-dev/Casper-server/repo/config" - fsrepo "github.com/Casper-dev/Casper-server/repo/fsrepo" - - homedir "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/mitchellh/go-homedir" + commands "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + corehttp "gitlab.com/casperDev/Casper-server/core/corehttp" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + config "gitlab.com/casperDev/Casper-server/repo/config" + fsrepo "gitlab.com/casperDev/Casper-server/repo/fsrepo" + + homedir "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/mitchellh/go-homedir" process "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" diff --git a/commands/cli/cmd_suggestion.go b/commands/cli/cmd_suggestion.go index 0487e2b..85c3f28 100644 --- a/commands/cli/cmd_suggestion.go +++ b/commands/cli/cmd_suggestion.go @@ -5,8 +5,8 @@ import ( "sort" "strings" - levenshtein "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/texttheater/golang-levenshtein/levenshtein" - cmds "github.com/Casper-dev/Casper-server/commands" + levenshtein "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/texttheater/golang-levenshtein/levenshtein" + cmds "gitlab.com/casperDev/Casper-server/commands" ) // Make a custom slice that can be sorted by its levenshtein value diff --git a/commands/cli/helptext.go b/commands/cli/helptext.go index cad5398..9c0b31d 100644 --- a/commands/cli/helptext.go +++ b/commands/cli/helptext.go @@ -7,7 +7,7 @@ import ( "strings" "text/template" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) const ( diff --git a/commands/cli/helptext_test.go b/commands/cli/helptext_test.go index 7fe3237..607ae39 100644 --- a/commands/cli/helptext_test.go +++ b/commands/cli/helptext_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) func TestSynopsisGenerator(t *testing.T) { diff --git a/commands/cli/parse.go b/commands/cli/parse.go index cc232ae..cda6266 100644 --- a/commands/cli/parse.go +++ b/commands/cli/parse.go @@ -9,8 +9,8 @@ import ( "sort" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - files "github.com/Casper-dev/Casper-server/commands/files" + cmds "gitlab.com/casperDev/Casper-server/commands" + files "gitlab.com/casperDev/Casper-server/commands/files" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" @@ -86,7 +86,15 @@ func ParseArgs(req cmds.Request, inputs []string, stdin *os.File, argDefs []cmds return nil, nil, u.ErrCast() } } - return parseArgs(inputs, stdin, argDefs, recursive, hidden, root) + + // TODO: use context? + if pwdOpt := req.Option("password"); pwdOpt != nil { + if pwd, pwdset, _ := pwdOpt.String(); pwdset { + return parseArgs(inputs, stdin, argDefs, recursive, hidden, root, []byte(pwd)) + } + } + + return parseArgs(inputs, stdin, argDefs, recursive, hidden, root, nil) } // Parse a command line made up of sub-commands, short arguments, long arguments and positional arguments @@ -256,7 +264,7 @@ func parseOpts(args []string, root *cmds.Command) ( const msgStdinInfo = "ipfs: Reading from %s; send Ctrl-d to stop." -func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) { +func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command, password []byte) ([]string, []files.File, error) { // ignore stdin on Windows if osh.IsWindows() { stdin = nil @@ -329,7 +337,13 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi fpath = stdin.Name() file = files.NewReaderFile("", fpath, r, nil) } else { - nf, err := appendFile(fpath, argDef, recursive, hidden) + var nf files.File + var err error + if password == nil { + nf, err = appendFile(fpath, argDef, recursive, hidden) + } else { + nf, err = appendFileEncrypted(fpath, argDef, recursive, hidden, password) + } if err != nil { return nil, nil, err } @@ -400,6 +414,10 @@ const dirNotSupportedFmtStr = "Invalid path '%s', argument '%s' does not support const winDriveLetterFmtStr = "%q is a drive letter, not a drive path" func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (files.File, error) { + return appendFileEncrypted(fpath, argDef, recursive, hidden, nil) +} + +func appendFileEncrypted(fpath string, argDef *cmds.Argument, recursive, hidden bool, password []byte) (files.File, error) { // resolve Windows relative dot paths like `X:.\somepath` if osh.IsWindows() { if len(fpath) >= 3 && fpath[1:3] == ":." { @@ -440,10 +458,10 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (fi } if osh.IsWindows() { - return windowsParseFile(fpath, hidden, stat) + return windowsParseFile(fpath, hidden, stat, password) } - return files.NewSerialFile(path.Base(fpath), fpath, hidden, stat) + return files.NewSerialFile(path.Base(fpath), fpath, hidden, stat, password) } // Inform the user if a file is waiting on input @@ -496,7 +514,7 @@ func (r *messageReader) Close() error { return r.r.Close() } -func windowsParseFile(fpath string, hidden bool, stat os.FileInfo) (files.File, error) { +func windowsParseFile(fpath string, hidden bool, stat os.FileInfo, password []byte) (files.File, error) { // special cases for Windows drive roots i.e. `X:\` and their long form `\\?\X:\` // drive path must be preserved as `X:\` (or it's longform) and not converted to `X:`, `X:.`, `\`, or `/` here switch len(fpath) { @@ -507,7 +525,7 @@ func windowsParseFile(fpath string, hidden bool, stat os.FileInfo) (files.File, } // `X:\` needs to preserve the `\`, path.Base(filepath.ToSlash(fpath)) results in `X:` which is not valid if fpath[1:3] == ":\\" { - return files.NewSerialFile(fpath, fpath, hidden, stat) + return files.NewSerialFile(fpath, fpath, hidden, stat, password) } case 6: // `\\?\X:` long prefix form of `X:`, still ambiguous @@ -518,9 +536,9 @@ func windowsParseFile(fpath string, hidden bool, stat os.FileInfo) (files.File, // `\\?\X:\` long prefix form is translated into short form `X:\` if fpath[:4] == "\\\\?\\" && fpath[5] == ':' && fpath[6] == '\\' { fpath = string(fpath[4]) + ":\\" - return files.NewSerialFile(fpath, fpath, hidden, stat) + return files.NewSerialFile(fpath, fpath, hidden, stat, password) } } - return files.NewSerialFile(path.Base(filepath.ToSlash(fpath)), fpath, hidden, stat) + return files.NewSerialFile(path.Base(filepath.ToSlash(fpath)), fpath, hidden, stat, password) } diff --git a/commands/cli/parse_test.go b/commands/cli/parse_test.go index b5a1967..f17be0d 100644 --- a/commands/cli/parse_test.go +++ b/commands/cli/parse_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/Casper-dev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/commands" ) type kvs map[string]interface{} diff --git a/commands/command.go b/commands/command.go index 2ee6feb..5b902cf 100644 --- a/commands/command.go +++ b/commands/command.go @@ -14,8 +14,9 @@ import ( "io" "reflect" - "github.com/Casper-dev/Casper-server/path" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + + "gitlab.com/casperDev/Casper-server/path" ) var log = logging.Logger("command") @@ -122,6 +123,7 @@ func (c *Command) Call(req Request) Response { isChan := false actualType := reflect.TypeOf(output) if actualType != nil { + log.Debugf("Actual type: %s", actualType.String()) if actualType.Kind() == reflect.Ptr { actualType = actualType.Elem() } diff --git a/commands/files/serialfile.go b/commands/files/serialfile.go index b17d14a..adf05bc 100644 --- a/commands/files/serialfile.go +++ b/commands/files/serialfile.go @@ -8,6 +8,8 @@ import ( "path/filepath" "strings" "syscall" + + "gitlab.com/casperDev/Casper-server/casper/crypto" ) // serialFile implements File, and reads from a path on the OS filesystem. @@ -19,10 +21,11 @@ type serialFile struct { files []os.FileInfo stat os.FileInfo current *File + password []byte handleHiddenFiles bool } -func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) { +func NewSerialFile(name, path string, hidden bool, stat os.FileInfo, passwd []byte) (File, error) { switch mode := stat.Mode(); { case mode.IsRegular(): @@ -30,6 +33,11 @@ func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, erro if err != nil { return nil, err } + + if passwd != nil { + fileEnc := crypto.NewAESEncryptReadCloser(file, passwd) + return NewReaderPathFile(name, path, fileEnc, stat) + } return NewReaderPathFile(name, path, file, stat) case mode.IsDir(): // for directories, stat all of the contents first, so we know what files to @@ -38,7 +46,8 @@ func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, erro if err != nil { return nil, err } - return &serialFile{name, path, contents, stat, nil, hidden}, nil + + return &serialFile{name, path, contents, stat, nil, passwd, hidden}, nil case mode&os.ModeSymlink != 0: target, err := os.Readlink(path) if err != nil { @@ -94,7 +103,7 @@ func (f *serialFile) NextFile() (File, error) { // recursively call the constructor on the next file // if it's a regular file, we will open it as a ReaderFile // if it's a directory, files in it will be opened serially - sf, err := NewSerialFile(fileName, filePath, f.handleHiddenFiles, stat) + sf, err := NewSerialFile(fileName, filePath, f.handleHiddenFiles, stat, f.password) if err != nil { return nil, err } diff --git a/commands/http/client.go b/commands/http/client.go index d640ab1..c9d13f8 100644 --- a/commands/http/client.go +++ b/commands/http/client.go @@ -12,8 +12,8 @@ import ( "strconv" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - config "github.com/Casper-dev/Casper-server/repo/config" + cmds "gitlab.com/casperDev/Casper-server/commands" + config "gitlab.com/casperDev/Casper-server/repo/config" context "context" ) diff --git a/commands/http/handler.go b/commands/http/handler.go index f3af27c..7a505c0 100644 --- a/commands/http/handler.go +++ b/commands/http/handler.go @@ -12,8 +12,8 @@ import ( "strings" "sync" - cmds "github.com/Casper-dev/Casper-server/commands" - "github.com/Casper-dev/Casper-server/repo/config" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/repo/config" cors "gx/ipfs/QmPG2kW5t27LuHgHnvhUwbHCNHAt2eUcb4gPHqofrESUdB/cors" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" @@ -190,7 +190,7 @@ func (i internalHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // now handle responding to the client properly - sendResponse(w, r, res, req) + SendResponse(w, r, res, req) } func guessMimeType(res cmds.Response) (string, error) { @@ -210,7 +210,7 @@ func guessMimeType(res cmds.Response) (string, error) { return mimeTypes[cmds.JSON], nil } -func sendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req cmds.Request) { +func SendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req cmds.Request) { h := w.Header() // Expose our agent to allow identification h.Set("Server", "go-ipfs/"+config.CurrentVersionNumber) @@ -280,14 +280,14 @@ func sendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req } w.WriteHeader(status) - err = flushCopy(w, out) + err = FlushCopy(w, out) if err != nil { log.Error("err: ", err) - w.Header().Set(StreamErrHeader, sanitizedErrStr(err)) + w.Header().Set(StreamErrHeader, SanitizedErrStr(err)) } } -func flushCopy(w io.Writer, r io.Reader) error { +func FlushCopy(w io.Writer, r io.Reader) error { buf := make([]byte, 4096) f, ok := w.(http.Flusher) if !ok { @@ -322,7 +322,7 @@ func flushCopy(w io.Writer, r io.Reader) error { } } -func sanitizedErrStr(err error) string { +func SanitizedErrStr(err error) string { s := err.Error() s = strings.Split(s, "\n")[0] s = strings.Split(s, "\r")[0] diff --git a/commands/http/handler_test.go b/commands/http/handler_test.go index 293586c..07ba685 100644 --- a/commands/http/handler_test.go +++ b/commands/http/handler_test.go @@ -6,9 +6,9 @@ import ( "net/url" "testing" - cmds "github.com/Casper-dev/Casper-server/commands" - ipfscmd "github.com/Casper-dev/Casper-server/core/commands" - coremock "github.com/Casper-dev/Casper-server/core/mock" + cmds "gitlab.com/casperDev/Casper-server/commands" + ipfscmd "gitlab.com/casperDev/Casper-server/core/commands" + coremock "gitlab.com/casperDev/Casper-server/core/mock" ) func assertHeaders(t *testing.T, resHeaders http.Header, reqHeaders map[string]string) { diff --git a/commands/http/multifilereader.go b/commands/http/multifilereader.go index 5312871..d53f41d 100644 --- a/commands/http/multifilereader.go +++ b/commands/http/multifilereader.go @@ -9,7 +9,7 @@ import ( "net/url" "sync" - files "github.com/Casper-dev/Casper-server/commands/files" + files "gitlab.com/casperDev/Casper-server/commands/files" ) // MultiFileReader reads from a `commands.File` (which can be a directory of files diff --git a/commands/http/multifilereader_test.go b/commands/http/multifilereader_test.go index a4df957..07c720e 100644 --- a/commands/http/multifilereader_test.go +++ b/commands/http/multifilereader_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - files "github.com/Casper-dev/Casper-server/commands/files" + files "gitlab.com/casperDev/Casper-server/commands/files" ) func TestOutput(t *testing.T) { diff --git a/commands/http/parse.go b/commands/http/parse.go index d6c963a..dbb8624 100644 --- a/commands/http/parse.go +++ b/commands/http/parse.go @@ -7,9 +7,9 @@ import ( "net/http" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - files "github.com/Casper-dev/Casper-server/commands/files" - path "github.com/Casper-dev/Casper-server/path" + cmds "gitlab.com/casperDev/Casper-server/commands" + files "gitlab.com/casperDev/Casper-server/commands/files" + path "gitlab.com/casperDev/Casper-server/path" ) // Parse parses the data in a http.Request and returns a command Request object diff --git a/commands/option.go b/commands/option.go index 1cf21ec..1cc4d7b 100644 --- a/commands/option.go +++ b/commands/option.go @@ -188,19 +188,26 @@ const ( RecLong = "recursive" ChanOpt = "stream-channels" TimeoutOpt = "timeout" + CallerOpt = "caller" + + CallerOptServer = "server" + CallerOptClient = "client" + CallerOptWeb = "web" ) // options that are used by this package var OptionEncodingType = StringOption(EncLong, EncShort, "The encoding type the output should be encoded with (json, xml, or text)") var OptionRecursivePath = BoolOption(RecLong, RecShort, "Add directory paths recursively").Default(false) var OptionStreamChannels = BoolOption(ChanOpt, "Stream channel output") -var OptionTimeout = StringOption(TimeoutOpt, "set a global timeout on the command") +var OptionTimeout = StringOption(TimeoutOpt, "Set a global timeout on the command") +var OptionCaller = StringOption(CallerOpt, "Request type (client or server)") // global options, added to every command var globalOptions = []Option{ OptionEncodingType, OptionStreamChannels, OptionTimeout, + OptionCaller, } // the above array of Options, wrapped in a Command diff --git a/commands/request.go b/commands/request.go index 365e40c..83ea003 100644 --- a/commands/request.go +++ b/commands/request.go @@ -11,10 +11,11 @@ import ( "strconv" "time" - "github.com/Casper-dev/Casper-server/commands/files" - "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/repo/config" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + + "gitlab.com/casperDev/Casper-server/commands/files" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/repo/config" ) type OptMap map[string]interface{} diff --git a/commands/response.go b/commands/response.go index 2c5b242..bfd22fd 100644 --- a/commands/response.go +++ b/commands/response.go @@ -176,6 +176,7 @@ func (r *response) Marshal() (io.Reader, error) { // Special case: if text encoding and an error, just print it out. if encType == Text && r.Error() != nil { + return strings.NewReader(r.Error().Error()), nil } diff --git a/core/bootstrap.go b/core/bootstrap.go index adecdb3..85b873e 100644 --- a/core/bootstrap.go +++ b/core/bootstrap.go @@ -8,10 +8,11 @@ import ( "sync" "time" - config "github.com/Casper-dev/Casper-server/repo/config" - math2 "github.com/Casper-dev/Casper-server/thirdparty/math2" lgbl "gx/ipfs/QmT4PgCNdv73hnFAqzHqwW44q7M9PWpykSswHDxndquZbc/go-libp2p-loggables" + config "gitlab.com/casperDev/Casper-server/repo/config" + math2 "gitlab.com/casperDev/Casper-server/thirdparty/math2" + context "context" inet "gx/ipfs/QmNa31VPzC561NWwRsJLE7nGYZYuuD2QfpK2b1q9BK54J1/go-libp2p-net" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" diff --git a/core/bootstrap_test.go b/core/bootstrap_test.go index f9172c6..60b7f6b 100644 --- a/core/bootstrap_test.go +++ b/core/bootstrap_test.go @@ -4,9 +4,10 @@ import ( "fmt" "testing" - config "github.com/Casper-dev/Casper-server/repo/config" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + + config "gitlab.com/casperDev/Casper-server/repo/config" ) func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { diff --git a/core/builder.go b/core/builder.go index 2bea035..a3ac825 100644 --- a/core/builder.go +++ b/core/builder.go @@ -9,16 +9,16 @@ import ( "syscall" "time" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - bserv "github.com/Casper-dev/Casper-server/blockservice" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - filestore "github.com/Casper-dev/Casper-server/filestore" - dag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" - pin "github.com/Casper-dev/Casper-server/pin" - repo "github.com/Casper-dev/Casper-server/repo" - cfg "github.com/Casper-dev/Casper-server/repo/config" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bserv "gitlab.com/casperDev/Casper-server/blockservice" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + filestore "gitlab.com/casperDev/Casper-server/filestore" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + pin "gitlab.com/casperDev/Casper-server/pin" + repo "gitlab.com/casperDev/Casper-server/repo" + cfg "gitlab.com/casperDev/Casper-server/repo/config" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" retry "gx/ipfs/QmPP91WFAb8LCs8EMzGvDPPvg1kacbqRkoxgTTnUsZckGe/retry-datastore" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" diff --git a/core/commands/active.go b/core/commands/active.go index 538305e..c896984 100644 --- a/core/commands/active.go +++ b/core/commands/active.go @@ -8,7 +8,7 @@ import ( "text/tabwriter" "time" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) var ActiveReqsCmd = &cmds.Command{ diff --git a/core/commands/add.go b/core/commands/add.go index a443c18..764678a 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -1,25 +1,36 @@ package commands import ( + "context" + "encoding/json" "errors" "fmt" "io" + "net" + "strconv" "strings" - - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - blockservice "github.com/Casper-dev/Casper-server/blockservice" - cmds "github.com/Casper-dev/Casper-server/commands" - files "github.com/Casper-dev/Casper-server/commands/files" - core "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/core/coreunix" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - dag "github.com/Casper-dev/Casper-server/merkledag" - dagtest "github.com/Casper-dev/Casper-server/merkledag/test" - mfs "github.com/Casper-dev/Casper-server/mfs" - ft "github.com/Casper-dev/Casper-server/unixfs" + "sync" + + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + "gitlab.com/casperDev/Casper-server/blockservice" + cu "gitlab.com/casperDev/Casper-server/casper/casper_utils" + "gitlab.com/casperDev/Casper-server/casper/uuid" + "gitlab.com/casperDev/Casper-server/client" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/commands/files" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core/coreunix" + "gitlab.com/casperDev/Casper-server/exchange/offline" + dag "gitlab.com/casperDev/Casper-server/merkledag" + dagtest "gitlab.com/casperDev/Casper-server/merkledag/test" + "gitlab.com/casperDev/Casper-server/mfs" + "gitlab.com/casperDev/Casper-server/pin" + ft "gitlab.com/casperDev/Casper-server/unixfs" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" + ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" "gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb" ) @@ -27,6 +38,9 @@ import ( var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded") const ( + RootObjectName = "" + finalObjectMarker = "" + quietOptionName = "quiet" quieterOptionName = "quieter" silentOptionName = "silent" @@ -42,6 +56,11 @@ const ( fstoreCacheOptionName = "fscache" cidVersionOptionName = "cid-version" hashOptionName = "hash" + uuidOptionName = "uuid" + passwordOptionName = "password" + updateOptionName = "update" + peersOptionName = "peers" + waitOptionName = "wait" ) const adderOutChanSize = 8 @@ -99,7 +118,7 @@ You can now check what blocks have been created by: }, Arguments: []cmds.Argument{ - cmds.FileArg("path", true, true, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(), + cmds.FileArg("path", true, false, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(), }, Options: []cmds.Option{ cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive) @@ -109,7 +128,7 @@ You can now check what blocks have been created by: cmds.BoolOption(progressOptionName, "p", "Stream progress data."), cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."), cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."), - cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."), + cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object.").Default(true), cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."), cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes] or rabin-[min]-[avg]-[max]").Default("size-262144"), cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true), @@ -118,6 +137,11 @@ You can now check what blocks have been created by: cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"), cmds.IntOption(cidVersionOptionName, "Cid version. Non-zero value will change default of 'raw-leaves' to true. (experimental)").Default(0), cmds.StringOption(hashOptionName, "Hash function to use. Will set Cid version to 1 if used. (experimental)").Default("sha2-256"), + cmds.StringOption(uuidOptionName, "Base58-encoded UUID to use. Generate random by default.").Default(nil), + cmds.BoolOption(updateOptionName, "Update file with existing UUID instead of adding new.").Default(true), + cmds.StringOption(passwordOptionName, "Encrypt files using password (AEC-256 CTR)."), + cmds.StringOption(peersOptionName, "JSON-encoded list of peer-multiaddrs").Default(""), + cmds.BoolOption(waitOptionName, "Wait until file is read").Default(""), }, PreRun: func(req cmds.Request) error { quiet, _, _ := req.Option(quietOptionName).Bool() @@ -148,6 +172,16 @@ You can now check what blocks have been created by: sizeCh := make(chan int64, 1) req.Values()["size"] = sizeCh + // On server we generate new UUID + + if _, uuidset, _ := req.Option(uuidOptionName).String(); uuidset { + // if UUID is specified on client, it is an update operation + caller, _, _ := req.Option(cmds.CallerOpt).String() + req.SetOption(updateOptionName, caller == cmds.CallerOptClient) + } else { + req.SetOption(uuidOptionName, base58.Encode(uuid.GenUUID())) + } + go func() { size, err := sizeFile.Size() if err != nil { @@ -195,6 +229,10 @@ You can now check what blocks have been created by: fscache, _, _ := req.Option(fstoreCacheOptionName).Bool() cidVer, _, _ := req.Option(cidVersionOptionName).Int() hashFunStr, hfset, _ := req.Option(hashOptionName).String() + caller, _, _ := req.Option(cmds.CallerOpt).String() + uuidOpt, _, _ := req.Option(uuidOptionName).String() + upd, _, _ := req.Option(updateOptionName).Bool() + //waitOpt, _, _ := req.Option(waitOptionName).Bool() if nocopy && !cfg.Experimental.FilestoreEnabled { res.SetError(errors.New("filestore is not enabled, see https://git.io/vy4XN"), @@ -293,6 +331,7 @@ You can now check what blocks have been created by: fileAdder.SetMfsRoot(mr) } + var root *dag.ProtoNode addAllAndPin := func(f files.File) error { // Iterate over each top-level file and add individually. Otherwise the // single files.File f is treated as a directory, affecting hidden file @@ -305,6 +344,7 @@ You can now check what blocks have been created by: } else if err != nil { return err } + if err := fileAdder.AddFile(file); err != nil { return err } @@ -320,7 +360,36 @@ You can now check what blocks have been created by: return nil } - return fileAdder.PinRoot() + // Set Root UUID as specified + dn, _ := fileAdder.RootNode() + pn := dn.(*dag.ProtoNode) + + uid := base58.Decode(uuidOpt) + log.Debugf("UUID: '%s'", uuidOpt) + if upd && caller == cmds.CallerOptClient { + newRoot, _ := pn.Copy().(*dag.ProtoNode) + newRoot.SetUUID(uid) + n.Pinning.PinWithMode(newRoot.Cid(), pin.Recursive) + exch.HasBlock(newRoot) + + uid = uuid.GenUUID() + } + + pn.SetUUID(uid) + exch.HasBlock(pn) + root = pn + n.AddUUID(uuidOpt, &core.UUIDInfo{PubKey: ""}) + + size, _ := pn.Size() + log.Debug(size) + outChan <- &coreunix.AddedObject{ + Name: RootObjectName, + Hash: pn.Cid().String(), + UUID: base58.Encode(pn.UUID()), + Size: strconv.FormatUint(size, 10), + } + + return fileAdder.PinRootUUID(uid) } go func() { @@ -329,7 +398,60 @@ You can now check what blocks have been created by: res.SetError(err, cmds.ErrNormal) return } + outChan <- &coreunix.AddedObject{ + Name: "", + Hash: finalObjectMarker, + UUID: "", + } + var performUpdate = func(_ ma.Multiaddr) {} + var peers []ma.Multiaddr + if popt, pf, _ := req.Option(peersOptionName).String(); pf { + var ps []string + err := json.Unmarshal([]byte(popt), &ps) + if err != nil { + log.Error(err) + return + } + for _, p := range ps { + peers = append(peers, ma.StringCast(p)) + } + performUpdate = func(peer ma.Multiaddr) { uploadRoot(req.Context(), n, peer, root) } + } else if caller == cmds.CallerOptClient { + if req.Option(updateOptionName).Found() { + log.Debugf("UUID is specified. Existing file will be updated %s %s", root.UUID(), root.Cid().String()) + b58uuid, _, _ := req.Option(uuidOptionName).String() + peers, err = cu.GetPeersMultiaddrsByHash(root.Cid().String()) + if err != nil { + log.Error(err) + return + } + performUpdate = func(peer ma.Multiaddr) { + updateRoot(req.Context(), n, peer, root, b58uuid) + } + } else { + size, _ := root.Size() + peers, err = cu.GetPeersMultiaddrsBySize(int64(size)) + fmt.Println("Got peeers", peers) + if err != nil { + log.Error(err) + return + } + performUpdate = func(peer ma.Multiaddr) { + uploadRoot(req.Context(), n, peer, root) + } + } + } + + wg := &sync.WaitGroup{} + wg.Add(len(peers)) + for _, peer := range peers { + go func() { + defer wg.Done() + performUpdate(peer) + }() + } + wg.Wait() }() }, PostRun: func(req cmds.Request, res cmds.Response) { @@ -374,26 +496,32 @@ You can now check what blocks have been created by: select { case out, ok := <-outChan: if !ok { + log.Debugf("Channel is closed") if quieter { fmt.Fprintln(res.Stdout(), lastHash) } break LOOP } output := out.(*coreunix.AddedObject) + if output.Hash == finalObjectMarker { + fmt.Println("Waiting for replication to finish...") + break + } + if len(output.Hash) > 0 { lastHash = output.Hash - if quieter { - continue - } - if progress { - // clear progress bar line before we print "added x" output - fmt.Fprintf(res.Stderr(), "\033[2K\r") - } - if quiet { - fmt.Fprintf(res.Stdout(), "%s\n", output.Hash) - } else { - fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name) + if !quieter { + if progress { + // clear progress bar line before we print "added x" output + fmt.Fprintf(res.Stderr(), "\033[2K\r") + } + + if quiet { + fmt.Fprintf(res.Stdout(), "%s\n", output.Hash) + } else { + fmt.Fprintf(res.Stdout(), "added %s %s %s %s\n", output.Hash, output.Name, output.UUID, output.Size) + } } } else { log.Debugf("add progress: %v %v\n", output.Name, output.Bytes) @@ -432,3 +560,43 @@ You can now check what blocks have been created by: }, Type: coreunix.AddedObject{}, } + +func uploadRoot(ctx context.Context, n *core.IpfsNode, peer ma.Multiaddr, root *dag.ProtoNode) { + err := n.ConnectToPeer(ctx, peer.String()) + if err != nil { + log.Error("Failed to connect: %s", err) + return + } + size, err := root.Size() + if err != nil { + log.Error("Cannot determine root size") + return + } + addr, _ := cu.MultiaddrToTCPAddr(peer) + + + thriftAddr := net.JoinHostPort(addr.IP.String(), "9090") + err = client.HandleClientUpload(ctx, thriftAddr, root.Cid().String(), int64(size), []string{}) + if err != nil { + log.Error("Error while uploading to %s: %s", thriftAddr, err) + } +} + +func updateRoot(ctx context.Context, n *core.IpfsNode, peer ma.Multiaddr, root *dag.ProtoNode, uuid string) { + err := n.ConnectToPeer(ctx, peer.String()) + if err != nil { + log.Error("Failed to connect: %s", err) + return + } + size, err := root.Size() + if err != nil { + log.Error("Cannot determine root size") + return + } + addr, _ := cu.MultiaddrToTCPAddr(peer) + thriftAddr := net.JoinHostPort(addr.IP.String(), "9090") + err = client.HandleClientUpdate(ctx, thriftAddr, uuid, root.Cid().String(), int64(size)) + if err != nil { + log.Error("Error while updating on %s: %s", peer, err) + } +} diff --git a/core/commands/bitswap.go b/core/commands/bitswap.go index 31a7dce..d9639f4 100644 --- a/core/commands/bitswap.go +++ b/core/commands/bitswap.go @@ -5,9 +5,9 @@ import ( "fmt" "io" - cmds "github.com/Casper-dev/Casper-server/commands" - bitswap "github.com/Casper-dev/Casper-server/exchange/bitswap" - decision "github.com/Casper-dev/Casper-server/exchange/bitswap/decision" + cmds "gitlab.com/casperDev/Casper-server/commands" + bitswap "gitlab.com/casperDev/Casper-server/exchange/bitswap" + decision "gitlab.com/casperDev/Casper-server/exchange/bitswap/decision" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" diff --git a/core/commands/block.go b/core/commands/block.go index 337f88f..e0f5711 100644 --- a/core/commands/block.go +++ b/core/commands/block.go @@ -7,22 +7,26 @@ import ( "io/ioutil" "strings" - util "github.com/Casper-dev/Casper-server/blocks/blockstore/util" - cmds "github.com/Casper-dev/Casper-server/commands" + bl "gitlab.com/casperDev/Casper-server/blocks" + util "gitlab.com/casperDev/Casper-server/blocks/blockstore/util" + uuid "gitlab.com/casperDev/Casper-server/casper/uuid" + cmds "gitlab.com/casperDev/Casper-server/commands" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" ) type BlockStat struct { Key string + UUID string Size int } func (bs BlockStat) String() string { - return fmt.Sprintf("Key: %s\nSize: %d\n", bs.Key, bs.Size) + return fmt.Sprintf("Key: %s\nSize: %d\nUUID: %s\n", bs.Key, bs.Size, bs.UUID) } var BlockCmd = &cmds.Command{ @@ -38,6 +42,7 @@ multihash. Subcommands: map[string]*cmds.Command{ "stat": blockStatCmd, "get": blockGetCmd, + "list": blockListCmd, "put": blockPutCmd, "rm": blockRmCmd, }, @@ -51,6 +56,7 @@ var blockStatCmd = &cmds.Command{ on raw IPFS blocks. It outputs the following to stdout: Key - the base58 encoded multihash + UUID - the base58 UUID Size - the size of the block in bytes `, @@ -66,8 +72,10 @@ on raw IPFS blocks. It outputs the following to stdout: return } + u, _ := bl.SplitData(b.RawData()) res.SetOutput(&BlockStat{ Key: b.Cid().String(), + UUID: base58.Encode(u), Size: len(b.RawData()), }) }, @@ -92,6 +100,7 @@ It outputs to stdout, and is a base58 encoded multihash. Arguments: []cmds.Argument{ cmds.StringArg("key", true, false, "The base58 multihash of an existing block to get.").EnableStdin(), }, + Type: []byte{}, Run: func(req cmds.Request, res cmds.Response) { b, err := getBlockForKey(req, req.Arguments()[0]) if err != nil { @@ -99,7 +108,74 @@ It outputs to stdout, and is a base58 encoded multihash. return } - res.SetOutput(bytes.NewReader(b.RawData())) + res.SetOutput(b.RawData()) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + bs := res.Output().([]byte) + u, d := bl.SplitData(bs) + return strings.NewReader(fmt.Sprintf("%s\n%s\n", base58.Encode(u), d)), nil + }, + }, +} + +type BlockHashList struct { + Hashes []string +} + +var blockListCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List hashes of all IPFS blocks in local repository.", + ShortDescription: ` +'ipfs block list' is a plumbing command for listing hashes of all IPFS blocks, +which are available locally. +`, + }, + Run: func(req cmds.Request, res cmds.Response) { + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + ch, err := n.Blockstore.AllKeysChan(req.Context()) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + var hashes []string + + LOOP: + for { + select { + case hash, ok := <-ch: + if !ok { + break LOOP + } + + hashes = append(hashes, hash.String()) + case <-req.Context().Done(): + res.SetError(req.Context().Err(), cmds.ErrNormal) + return + } + } + + res.SetOutput(&BlockHashList{Hashes: hashes}) + }, + Type: BlockHashList{}, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + hashList, ok := res.Output().(*BlockHashList) + if !ok { + return nil, u.ErrCast() + } + out := new(bytes.Buffer) + for _, h := range hashList.Hashes { + fmt.Fprintf(out, "%s\n", h) + } + return out, nil + }, }, } @@ -114,6 +190,7 @@ It reads from stdin, and is a base58 encoded multihash. Arguments: []cmds.Argument{ cmds.FileArg("data", true, false, "The data to be stored as an IPFS block.").EnableStdin(), + cmds.StringArg("uuid", false, false, "The base58 UUID of an existing block to replace."), }, Options: []cmds.Option{ cmds.StringOption("format", "f", "cid format for blocks to be created with.").Default("v0"), @@ -174,13 +251,23 @@ It reads from stdin, and is a base58 encoded multihash. } pref.MhLength = mhlen - bcid, err := pref.Sum(data) + var id []byte + if len(req.StringArguments()) > 0 { + id = base58.Decode(req.StringArguments()[0]) + } else { + id = uuid.GenUUID() + } + + b58uuid := base58.Encode(id) + log.Debugf("UUID for new block: %s", b58uuid) + + bcid, err := pref.Sum(id) if err != nil { res.SetError(err, cmds.ErrNormal) return } - b, err := blocks.NewBlockWithCid(data, bcid) + b, err := bl.NewBlockWithCid(append(id, data...), bcid) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -195,13 +282,14 @@ It reads from stdin, and is a base58 encoded multihash. res.SetOutput(&BlockStat{ Key: k.String(), + UUID: b58uuid, Size: len(data), }) }, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { bs := res.Output().(*BlockStat) - return strings.NewReader(bs.Key + "\n"), nil + return strings.NewReader(bs.UUID + "\n" + bs.Key + "\n"), nil }, }, Type: BlockStat{}, diff --git a/core/commands/bootstrap.go b/core/commands/bootstrap.go index 3d21ffd..795fe82 100644 --- a/core/commands/bootstrap.go +++ b/core/commands/bootstrap.go @@ -6,11 +6,12 @@ import ( "io" "sort" - cmds "github.com/Casper-dev/Casper-server/commands" - repo "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - "github.com/Casper-dev/Casper-server/repo/fsrepo" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + + cmds "gitlab.com/casperDev/Casper-server/commands" + repo "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + "gitlab.com/casperDev/Casper-server/repo/fsrepo" ) type BootstrapOutput struct { diff --git a/core/commands/cat.go b/core/commands/cat.go index a6b79fe..4e9a66c 100644 --- a/core/commands/cat.go +++ b/core/commands/cat.go @@ -1,17 +1,28 @@ package commands import ( + "context" + "fmt" "io" + "net" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" + cu "gitlab.com/casperDev/Casper-server/casper/casper_utils" + "gitlab.com/casperDev/Casper-server/casper/crypto" + "gitlab.com/casperDev/Casper-server/client" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" - context "context" + "gitlab.com/casperDev/Casper-SC/casper_sc" + + "gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb" ) const progressBarMinSize = 1024 * 1024 * 8 // show progress bar for outputs > 8MiB +// defined in commands/add +//const passwordOptionName = "password" + var CatCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Show IPFS object data.", @@ -19,7 +30,10 @@ var CatCmd = &cmds.Command{ }, Arguments: []cmds.Argument{ - cmds.StringArg("ipfs-path", true, true, "The path to the IPFS object(s) to be outputted.").EnableStdin(), + cmds.StringArg("ipfs-path", true, false, "The path to the IPFS object(s) to be outputted.").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.StringOption(passwordOptionName, "Password decryption key"), }, Run: func(req cmds.Request, res cmds.Response) { node, err := req.InvocContext().GetNode() @@ -28,6 +42,32 @@ var CatCmd = &cmds.Command{ return } + caller, _, _ := req.Option(cmds.CallerOpt).String() + if caller == cmds.CallerOptClient { + _, _, auth, _ := Casper_SC.GetSC() + wallet := auth.From.String() + hash := req.Arguments()[0] + peers, err := cu.GetPeersMultiaddrsByHash(hash) + if err != nil && len(peers) == 0 { + res.SetError(err, cmds.ErrClient) + return + } + for _, peer := range peers { + err := node.ConnectToPeer(req.Context(), peer.String()) + if err != nil { + log.Error("Failed to connect: %s", err) + continue + } + addr, _ := cu.MultiaddrToTCPAddr(peer) + thriftAddr := net.JoinHostPort(addr.String(), "9090") + err = client.HandleClientDownload(req.Context(), thriftAddr, hash, wallet) + if err == nil { + break + } + } + fmt.Println("Success!") + } + if !node.OnlineMode() { if err := node.SetupOfflineRouting(); err != nil { res.SetError(err, cmds.ErrNormal) @@ -49,17 +89,24 @@ var CatCmd = &cmds.Command{ */ res.SetLength(length) - reader := io.MultiReader(readers...) + res.SetOutput(reader) }, PostRun: func(req cmds.Request, res cmds.Response) { - if res.Length() < progressBarMinSize { + if res.Error() != nil { return } + reader := res.Output().(io.Reader) + if password, pwdset, _ := req.Option(passwordOptionName).String(); pwdset { + reader = crypto.NewAESReader(reader, []byte(password)) + } - bar, reader := progressBarForReader(res.Stderr(), res.Output().(io.Reader), int64(res.Length())) - bar.Start() + var bar *pb.ProgressBar + if res.Length() >= progressBarMinSize { + bar, reader = progressBarForReader(res.Stderr(), reader, int64(res.Length())) + bar.Start() + } res.SetOutput(reader) }, diff --git a/core/commands/commands.go b/core/commands/commands.go index adea1e2..c4b1006 100644 --- a/core/commands/commands.go +++ b/core/commands/commands.go @@ -11,7 +11,7 @@ import ( "sort" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) type Command struct { diff --git a/core/commands/config.go b/core/commands/config.go index b47f877..cb25dd8 100644 --- a/core/commands/config.go +++ b/core/commands/config.go @@ -11,10 +11,10 @@ import ( "os/exec" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - repo "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - fsrepo "github.com/Casper-dev/Casper-server/repo/fsrepo" + cmds "gitlab.com/casperDev/Casper-server/commands" + repo "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + fsrepo "gitlab.com/casperDev/Casper-server/repo/fsrepo" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" ) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 28362ba..ec3d64f 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -7,14 +7,20 @@ import ( "math" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - files "github.com/Casper-dev/Casper-server/commands/files" - coredag "github.com/Casper-dev/Casper-server/core/coredag" - path "github.com/Casper-dev/Casper-server/path" - pin "github.com/Casper-dev/Casper-server/pin" + "gitlab.com/casperDev/Casper-server/casper/uuid" + cval "gitlab.com/casperDev/Casper-server/casper/validation" + cmds "gitlab.com/casperDev/Casper-server/commands" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + + files "gitlab.com/casperDev/Casper-server/commands/files" + coredag "gitlab.com/casperDev/Casper-server/core/coredag" + path "gitlab.com/casperDev/Casper-server/path" + pin "gitlab.com/casperDev/Casper-server/pin" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" ) @@ -29,9 +35,11 @@ to deprecate and replace the existing 'ipfs object' command moving forward. `, }, Subcommands: map[string]*cmds.Command{ - "put": DagPutCmd, - "get": DagGetCmd, - "resolve": DagResolveCmd, + "put": DagPutCmd, + "get": DagGetCmd, + "resolve": DagResolveCmd, + "checksum": DagChecksumCmd, + "stat": DagStatCmd, }, } @@ -183,6 +191,167 @@ into an object of the specified format. }, } +const ( + startOptionName = "start" + stopOptionName = "stop" + saltOptionName = "salt" + uuidOptionName = "uuid" +) + +var DagChecksumCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Calculate checksum for a range of file.", + ShortDescription: ` +'ipfs dag checksum' calculates sha256-checksum for a range of a file node +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("ref", true, false, "Hash or UUID of file"), + }, + // TODO: uint64 option + Options: []cmds.Option{ + cmds.IntOption(startOptionName, "First byte number.").Default(0), + cmds.IntOption(stopOptionName, "Last byte number. If 0, read till end.").Default(0), + cmds.StringOption(saltOptionName, "Salt to add before hashing.").Default(""), + cmds.BoolOption(uuidOptionName, "Assume that ref is UUID.").Default(false), + }, + Run: func(req cmds.Request, res cmds.Response) { + start, _, _ := req.Option(startOptionName).Int() + stop, _, _ := req.Option(stopOptionName).Int() + salt, _, _ := req.Option(saltOptionName).String() + isUUID, _, _ := req.Option(uuidOptionName).Bool() + + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + id := req.Arguments()[0] + if isUUID { + id = uuid.UUIDToHash(base58.Decode(req.Arguments()[0])).B58String() + } + + p, err := path.ParsePath(id) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + obj, _, err := n.Resolver.ResolveToLastNode(req.Context(), p) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + cs, err := cval.ChecksumSalt(req.Context(), obj, int64(start), int64(stop), n.Resolver.DAG, []byte(salt)) + if err != nil { + res.SetError(err, cmds.ErrNormal) + } + + //client.InvokeGetFileChecksum(req.Context(), "10.10.10.1:9090", req.Arguments()[0], int64(start), int64(stop), salt) + + res.SetOutput(strings.NewReader(cs.B58String() + "\n")) + }, +} + +type DagStat struct { + Name string + Size uint64 + UUID string + Hash string +} + +var DagStatCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Get info about file.", + ShortDescription: ` +'ipfs dag stat' returns file information, such as size, name etc. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("ref", true, false, "Hash or UUID of file"), + }, + // TODO: uint64 option + Options: []cmds.Option{ + cmds.BoolOption(uuidOptionName, "Assume that ref is UUID.").Default(false), + }, + Type: &DagStat{}, + Run: func(req cmds.Request, res cmds.Response) { + isUUID, _, _ := req.Option(uuidOptionName).Bool() + + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + id := req.Arguments()[0] + if isUUID { + id = uuid.UUIDToHash(base58.Decode(req.Arguments()[0])).B58String() + } + + p, err := path.ParsePath(id) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + obj, _, err := n.Resolver.ResolveToLastNode(req.Context(), p) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + stat := &DagStat{} + switch v := obj.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(v.Data()) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + if fsn.Type == ft.TDirectory && len(fsn.Data) == 0 && len(v.Links()) == 1 { + // this directory is wrapped over one file + // return slice of that file + stat.Name = v.Links()[0].Name + stat.UUID = base58.Encode(v.UUID()) + stat.Hash = v.Cid().String() + + child, err := v.Links()[0].GetNode(req.Context(), n.DAG) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + switch c := child.(type) { + case *dag.ProtoNode: + fsn, err := ft.FSNodeFromBytes(c.Data()) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + stat.Size = fsn.FileSize() + case *dag.RawNode: + s, err := c.Stat() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + stat.Size = uint64(s.DataSize) + default: + res.SetError(err, cmds.ErrNormal) + return + } + } + default: + res.SetError(fmt.Errorf("Not a DAG node."), cmds.ErrNormal) + } + + res.SetOutput(&stat) + }, +} + var DagGetCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Get a dag node from ipfs.", @@ -225,6 +394,12 @@ format. res.SetOutput(out) }, + Marshalers: cmds.MarshalerMap{ + ///cmds.Text: func(res cmds.Response) (io.Reader, error) { + /// output := res.Output().(node.Node) + /// return bytes.NewReader(output.RawData()), nil + ///}, + }, } // DagResolveCmd returns address of highest block within a path and a path remainder diff --git a/core/commands/del.go b/core/commands/del.go new file mode 100644 index 0000000..d8f84cd --- /dev/null +++ b/core/commands/del.go @@ -0,0 +1,81 @@ +package commands + +import ( + "fmt" + "math/big" + "net" + + "gitlab.com/casperDev/Casper-SC/casper_sc" + util "gitlab.com/casperDev/Casper-server/blocks/blockstore/util" + cu "gitlab.com/casperDev/Casper-server/casper/casper_utils" + "gitlab.com/casperDev/Casper-server/client" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core/corerepo" +) + +var DelCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove block from server.", + ShortDescription: "TODO", + }, + Arguments: []cmds.Argument{ + cmds.StringArg("ipfs-path", true, false, "The path to the IPFS object to be removed.").EnableStdin(), + }, + Run: func(req cmds.Request, res cmds.Response) { + caller, _, _ := req.Option(cmds.CallerOpt).String() + if caller == cmds.CallerOptWeb { + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + removed, err := corerepo.Unpin(n, req.Context(), req.Arguments(), true) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + ch, err := util.RmBlocks(n.Blockstore, n.Pinning, removed, util.RmBlocksOpts{ + Quiet: true, + Force: true, + }) + log.Debugf("Removed %d blocks", len(removed)) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + // wait for delete to finish + for _ = range ch { + } + return + } + + casperSClient, _, _, _ := Casper_SC.GetSC() + casperSClient.NotifyDelete(nil, big.NewInt(int64(1337))) + hash := req.Arguments()[0] + peers, err := cu.GetPeersMultiaddrsByHash(hash) + if err != nil && len(peers) == 0 { + res.SetError(err, cmds.ErrNormal) + return + } + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + for _, peer := range peers { + err := n.ConnectToPeer(req.Context(), peer.String()) + if err != nil { + log.Error("Failed to connect: %s", err) + continue + } + addr, _ := cu.MultiaddrToTCPAddr(peer) + thriftAddr := net.JoinHostPort(addr.IP.String(), "9090") + err = client.HandleClientDelete(req.Context(), thriftAddr, hash) + if err != nil { + log.Errorf("Error while deleting file from peer '%s'", peer) + } + } + + fmt.Println("Success!") + }, +} diff --git a/core/commands/dht.go b/core/commands/dht.go index bb7b07a..db273f1 100644 --- a/core/commands/dht.go +++ b/core/commands/dht.go @@ -8,9 +8,9 @@ import ( "io" "time" - cmds "github.com/Casper-dev/Casper-server/commands" - dag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" + cmds "gitlab.com/casperDev/Casper-server/commands" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" diff --git a/core/commands/dht_test.go b/core/commands/dht_test.go index 1b5f876..693a6ee 100644 --- a/core/commands/dht_test.go +++ b/core/commands/dht_test.go @@ -3,8 +3,9 @@ package commands import ( "testing" - "github.com/Casper-dev/Casper-server/namesys" tu "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + + "gitlab.com/casperDev/Casper-server/namesys" ) func TestKeyTranslation(t *testing.T) { diff --git a/core/commands/diag.go b/core/commands/diag.go index d6864c2..9a0877a 100644 --- a/core/commands/diag.go +++ b/core/commands/diag.go @@ -1,6 +1,6 @@ package commands -import cmds "github.com/Casper-dev/Casper-server/commands" +import cmds "gitlab.com/casperDev/Casper-server/commands" var DiagCmd = &cmds.Command{ Helptext: cmds.HelpText{ diff --git a/core/commands/dns.go b/core/commands/dns.go index 5f117c4..ea92826 100644 --- a/core/commands/dns.go +++ b/core/commands/dns.go @@ -4,9 +4,10 @@ import ( "io" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - namesys "github.com/Casper-dev/Casper-server/namesys" util "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + + cmds "gitlab.com/casperDev/Casper-server/commands" + namesys "gitlab.com/casperDev/Casper-server/namesys" ) var DNSCmd = &cmds.Command{ diff --git a/core/commands/external.go b/core/commands/external.go index f4a0c31..d61e316 100644 --- a/core/commands/external.go +++ b/core/commands/external.go @@ -8,7 +8,7 @@ import ( "os/exec" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) func ExternalBinary() *cmds.Command { diff --git a/core/commands/files/files.go b/core/commands/files/files.go index 487e521..6789f16 100644 --- a/core/commands/files/files.go +++ b/core/commands/files/files.go @@ -10,13 +10,13 @@ import ( gopath "path" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - dag "github.com/Casper-dev/Casper-server/merkledag" - mfs "github.com/Casper-dev/Casper-server/mfs" - path "github.com/Casper-dev/Casper-server/path" - ft "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mfs "gitlab.com/casperDev/Casper-server/mfs" + path "gitlab.com/casperDev/Casper-server/path" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/commands/filestore.go b/core/commands/filestore.go index 425650e..412883c 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -5,11 +5,12 @@ import ( "fmt" "io" - cmds "github.com/Casper-dev/Casper-server/commands" - "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/filestore" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/filestore" ) var FileStoreCmd = &cmds.Command{ diff --git a/core/commands/get.go b/core/commands/get.go index f84bae2..7d5c87a 100644 --- a/core/commands/get.go +++ b/core/commands/get.go @@ -5,18 +5,21 @@ import ( "errors" "fmt" "io" + "net" "os" - gopath "path" "strings" - "gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb" + "gitlab.com/casperDev/Casper-SC/casper_sc" + cu "gitlab.com/casperDev/Casper-server/casper/casper_utils" + "gitlab.com/casperDev/Casper-server/client" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core" + dag "gitlab.com/casperDev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/path" + "gitlab.com/casperDev/Casper-server/thirdparty/tar" + uarchive "gitlab.com/casperDev/Casper-server/unixfs/archive" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - dag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" - tar "github.com/Casper-dev/Casper-server/thirdparty/tar" - uarchive "github.com/Casper-dev/Casper-server/unixfs/archive" + "gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb" ) var ErrInvalidCompressionLevel = errors.New("Compression level must be between 1 and 9") @@ -45,6 +48,7 @@ may also specify the level of compression by specifying '-l=<1-9>'. cmds.BoolOption("archive", "a", "Output a TAR archive.").Default(false), cmds.BoolOption("compress", "C", "Compress the output with GZIP compression.").Default(false), cmds.IntOption("compression-level", "l", "The level of compression (1-9).").Default(-1), + cmds.StringOption(passwordOptionName, "Decrypt file(s) using password."), }, PreRun: func(req cmds.Request) error { _, err := getCompressOptions(req) @@ -67,6 +71,35 @@ may also specify the level of compression by specifying '-l=<1-9>'. res.SetError(err, cmds.ErrNormal) return } + + caller, _, _ := req.Option(cmds.CallerOpt).String() + if caller == cmds.CallerOptClient { + _, _, auth, _ := Casper_SC.GetSC() + wallet := auth.From.String() + firstHash := req.Arguments()[0] + fmt.Println(firstHash) + + peers, err := cu.GetPeersMultiaddrsByHash(firstHash) + if err != nil && len(peers) == 0 { + res.SetError(err, cmds.ErrNormal) + return + } + for _, peer := range peers { + err := node.ConnectToPeer(req.Context(), peer.String()) + if err != nil { + log.Error("Failed to connect: %s", err) + continue + } + addr, _ := cu.MultiaddrToTCPAddr(peer) + thriftAddr := net.JoinHostPort(addr.IP.String(), "9090") + err = client.HandleClientDownload(req.Context(), thriftAddr, firstHash, wallet) + if err == nil { + break + } + } + fmt.Println("Success!") + } + p := path.Path(req.Arguments()[0]) ctx := req.Context() dn, err := core.Resolve(ctx, node.Namesys, node.Resolver, p) @@ -92,6 +125,7 @@ may also specify the level of compression by specifying '-l=<1-9>'. } archive, _, _ := req.Option("archive").Bool() + reader, err := uarchive.DagArchive(ctx, dn, p.String(), node.DAG, archive, cmplvl) if err != nil { res.SetError(err, cmds.ErrNormal) @@ -106,10 +140,14 @@ may also specify the level of compression by specifying '-l=<1-9>'. outReader := res.Output().(io.Reader) res.SetOutput(nil) + var err error outPath, _, _ := req.Option("output").String() if len(outPath) == 0 { - _, outPath = gopath.Split(req.Arguments()[0]) - outPath = gopath.Clean(outPath) + outPath, err = os.Getwd() + if err != nil { + res.SetError(err, cmds.ErrClient) + return + } } cmplvl, err := getCompressOptions(req) @@ -119,12 +157,17 @@ may also specify the level of compression by specifying '-l=<1-9>'. } archive, _, _ := req.Option("archive").Bool() + var password []byte + if pwd, pf, _ := req.Option(passwordOptionName).String(); pf { + password = []byte(pwd) + } gw := getWriter{ Out: os.Stdout, Err: os.Stderr, Archive: archive, Compression: cmplvl, + Password: password, Size: int64(res.Length()), } @@ -177,6 +220,7 @@ type getWriter struct { Archive bool Compression int + Password []byte Size int64 } @@ -225,7 +269,11 @@ func (gw *getWriter) writeExtracted(r io.Reader, fpath string) error { defer bar.Finish() defer bar.Set64(gw.Size) - extractor := &tar.Extractor{fpath, bar.Add64} + extractor := &tar.Extractor{ + Password: gw.Password, + Path: fpath, + Progress: bar.Add64, + } return extractor.Extract(r) } diff --git a/core/commands/helptext_test.go b/core/commands/helptext_test.go index b0d0528..5d3cce9 100644 --- a/core/commands/helptext_test.go +++ b/core/commands/helptext_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) func checkHelptextRecursive(t *testing.T, name []string, c *cmds.Command) { diff --git a/core/commands/id.go b/core/commands/id.go index 2a7f036..09b0d99 100644 --- a/core/commands/id.go +++ b/core/commands/id.go @@ -10,10 +10,11 @@ import ( b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" kb "gx/ipfs/QmSAFA8v42u4gpJNy1tb7vW3JiiXiaYDC2b845c2RnNSJL/go-libp2p-kbucket" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" diff --git a/core/commands/ipns.go b/core/commands/ipns.go index 4db969e..ace7fb2 100644 --- a/core/commands/ipns.go +++ b/core/commands/ipns.go @@ -5,10 +5,11 @@ import ( "io" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - namesys "github.com/Casper-dev/Casper-server/namesys" - offline "github.com/Casper-dev/Casper-server/routing/offline" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + + cmds "gitlab.com/casperDev/Casper-server/commands" + namesys "gitlab.com/casperDev/Casper-server/namesys" + offline "gitlab.com/casperDev/Casper-server/routing/offline" ) var IpnsCmd = &cmds.Command{ diff --git a/core/commands/keystore.go b/core/commands/keystore.go index 3b84083..627f83d 100644 --- a/core/commands/keystore.go +++ b/core/commands/keystore.go @@ -10,7 +10,7 @@ import ( "strings" "text/tabwriter" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" diff --git a/core/commands/log.go b/core/commands/log.go index b033b84..f2a5797 100644 --- a/core/commands/log.go +++ b/core/commands/log.go @@ -6,7 +6,7 @@ import ( logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) // Golang os.Args overrides * and replaces the character argument with diff --git a/core/commands/ls.go b/core/commands/ls.go index 75933b8..65d88eb 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -6,15 +6,15 @@ import ( "io" "text/tabwriter" - blockservice "github.com/Casper-dev/Casper-server/blockservice" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - merkledag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" - unixfs "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" - unixfspb "github.com/Casper-dev/Casper-server/unixfs/pb" + blockservice "gitlab.com/casperDev/Casper-server/blockservice" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + merkledag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + unixfs "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + unixfspb "gitlab.com/casperDev/Casper-server/unixfs/pb" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/core/commands/mount_nofuse.go b/core/commands/mount_nofuse.go index 73044af..b7ba379 100644 --- a/core/commands/mount_nofuse.go +++ b/core/commands/mount_nofuse.go @@ -4,7 +4,7 @@ package commands import ( - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) var MountCmd = &cmds.Command{ diff --git a/core/commands/mount_unix.go b/core/commands/mount_unix.go index aa7eea3..3b9135f 100644 --- a/core/commands/mount_unix.go +++ b/core/commands/mount_unix.go @@ -8,9 +8,9 @@ import ( "io" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - nodeMount "github.com/Casper-dev/Casper-server/fuse/node" - config "github.com/Casper-dev/Casper-server/repo/config" + cmds "gitlab.com/casperDev/Casper-server/commands" + nodeMount "gitlab.com/casperDev/Casper-server/fuse/node" + config "gitlab.com/casperDev/Casper-server/repo/config" ) var MountCmd = &cmds.Command{ diff --git a/core/commands/mount_windows.go b/core/commands/mount_windows.go index cfe3338..99bdb03 100644 --- a/core/commands/mount_windows.go +++ b/core/commands/mount_windows.go @@ -3,7 +3,7 @@ package commands import ( "errors" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) var MountCmd = &cmds.Command{ diff --git a/core/commands/name.go b/core/commands/name.go index 80bf92f..09641d5 100644 --- a/core/commands/name.go +++ b/core/commands/name.go @@ -1,6 +1,6 @@ package commands -import cmds "github.com/Casper-dev/Casper-server/commands" +import cmds "gitlab.com/casperDev/Casper-server/commands" type IpnsEntry struct { Name string diff --git a/core/commands/object/diff.go b/core/commands/object/diff.go index 1952f7a..ca0dcd9 100644 --- a/core/commands/object/diff.go +++ b/core/commands/object/diff.go @@ -5,10 +5,10 @@ import ( "fmt" "io" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - dagutils "github.com/Casper-dev/Casper-server/merkledag/utils" - path "github.com/Casper-dev/Casper-server/path" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + dagutils "gitlab.com/casperDev/Casper-server/merkledag/utils" + path "gitlab.com/casperDev/Casper-server/path" ) type Changes struct { diff --git a/core/commands/object/object.go b/core/commands/object/object.go index 676b7f1..44c81ea 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -12,12 +12,12 @@ import ( "strings" "text/tabwriter" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - dag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" - pin "github.com/Casper-dev/Casper-server/pin" - ft "github.com/Casper-dev/Casper-server/unixfs" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + pin "gitlab.com/casperDev/Casper-server/pin" + ft "gitlab.com/casperDev/Casper-server/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index e46b820..4140ad2 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -5,13 +5,14 @@ import ( "io/ioutil" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - dag "github.com/Casper-dev/Casper-server/merkledag" - dagutils "github.com/Casper-dev/Casper-server/merkledag/utils" - path "github.com/Casper-dev/Casper-server/path" - ft "github.com/Casper-dev/Casper-server/unixfs" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + dag "gitlab.com/casperDev/Casper-server/merkledag" + dagutils "gitlab.com/casperDev/Casper-server/merkledag/utils" + path "gitlab.com/casperDev/Casper-server/path" + ft "gitlab.com/casperDev/Casper-server/unixfs" ) var ObjectPatchCmd = &cmds.Command{ diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 3f34f02..78f7cf8 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -8,8 +8,8 @@ import ( "strconv" "text/tabwriter" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" ) diff --git a/core/commands/pin.go b/core/commands/pin.go index 9cb831f..597b0b9 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -6,13 +6,13 @@ import ( "io" "time" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - corerepo "github.com/Casper-dev/Casper-server/core/corerepo" - dag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" - pin "github.com/Casper-dev/Casper-server/pin" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + corerepo "gitlab.com/casperDev/Casper-server/core/corerepo" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + pin "gitlab.com/casperDev/Casper-server/pin" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" context "context" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" diff --git a/core/commands/ping.go b/core/commands/ping.go index 14506f4..6e48539 100644 --- a/core/commands/ping.go +++ b/core/commands/ping.go @@ -8,12 +8,13 @@ import ( "strings" "time" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + context "context" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" ) diff --git a/core/commands/publish.go b/core/commands/publish.go index be4822f..ac61f08 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -8,10 +8,10 @@ import ( "strings" "time" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - keystore "github.com/Casper-dev/Casper-server/keystore" - path "github.com/Casper-dev/Casper-server/path" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + keystore "gitlab.com/casperDev/Casper-server/keystore" + path "gitlab.com/casperDev/Casper-server/path" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" crypto "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go index 8a0482f..dc91b74 100644 --- a/core/commands/pubsub.go +++ b/core/commands/pubsub.go @@ -10,10 +10,11 @@ import ( "sync" "time" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/core/commands/refs.go b/core/commands/refs.go index ceee2e0..f264541 100644 --- a/core/commands/refs.go +++ b/core/commands/refs.go @@ -7,10 +7,10 @@ import ( "io" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - "github.com/Casper-dev/Casper-server/core" - dag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/commands/repo.go b/core/commands/repo.go index 4200d97..52ade00 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -9,12 +9,12 @@ import ( "strings" "text/tabwriter" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - cmds "github.com/Casper-dev/Casper-server/commands" - corerepo "github.com/Casper-dev/Casper-server/core/corerepo" - config "github.com/Casper-dev/Casper-server/repo/config" - fsrepo "github.com/Casper-dev/Casper-server/repo/fsrepo" - lockfile "github.com/Casper-dev/Casper-server/repo/fsrepo/lock" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + cmds "gitlab.com/casperDev/Casper-server/commands" + corerepo "gitlab.com/casperDev/Casper-server/core/corerepo" + config "gitlab.com/casperDev/Casper-server/repo/config" + fsrepo "gitlab.com/casperDev/Casper-server/repo/fsrepo" + lockfile "gitlab.com/casperDev/Casper-server/repo/fsrepo/lock" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/core/commands/resolve.go b/core/commands/resolve.go index afc5716..07faf3b 100644 --- a/core/commands/resolve.go +++ b/core/commands/resolve.go @@ -4,11 +4,12 @@ import ( "io" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - "github.com/Casper-dev/Casper-server/core" - ns "github.com/Casper-dev/Casper-server/namesys" - path "github.com/Casper-dev/Casper-server/path" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/core" + ns "gitlab.com/casperDev/Casper-server/namesys" + path "gitlab.com/casperDev/Casper-server/path" ) type ResolvedPath struct { diff --git a/core/commands/root.go b/core/commands/root.go index 539d274..8bb5e3b 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -4,12 +4,13 @@ import ( "io" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - dag "github.com/Casper-dev/Casper-server/core/commands/dag" - files "github.com/Casper-dev/Casper-server/core/commands/files" - ocmd "github.com/Casper-dev/Casper-server/core/commands/object" - unixfs "github.com/Casper-dev/Casper-server/core/commands/unixfs" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + + cmds "gitlab.com/casperDev/Casper-server/commands" + dag "gitlab.com/casperDev/Casper-server/core/commands/dag" + files "gitlab.com/casperDev/Casper-server/core/commands/files" + ocmd "gitlab.com/casperDev/Casper-server/core/commands/object" + unixfs "gitlab.com/casperDev/Casper-server/core/commands/unixfs" ) var log = logging.Logger("core/commands") @@ -27,9 +28,11 @@ BASIC COMMANDS init Initialize ipfs local configuration add Add a file to IPFS cat Show IPFS object data + del Delete IPFS object from server and locally get Download IPFS objects ls List links from an object refs List hashes of links from an object + upd Update reference to object by UUID DATA STRUCTURE COMMANDS block Interact with raw blocks in the datastore @@ -98,6 +101,7 @@ var rootSubcommands = map[string]*cmds.Command{ "block": BlockCmd, "bootstrap": BootstrapCmd, "cat": CatCmd, + "del": DelCmd, "commands": CommandsDaemonCmd, "config": ConfigCmd, "dag": dag.DagCmd, @@ -125,9 +129,11 @@ var rootSubcommands = map[string]*cmds.Command{ "tar": TarCmd, "file": unixfs.UnixFSCmd, "update": ExternalBinary(), + "upd": UpdCmd, "version": VersionCmd, "bitswap": BitswapCmd, "filestore": FileStoreCmd, + "validate": ValidateCmd, "shutdown": daemonShutdownCmd, } @@ -141,6 +147,7 @@ var RefsROCmd = &cmds.Command{} var rootROSubcommands = map[string]*cmds.Command{ "block": &cmds.Command{ Subcommands: map[string]*cmds.Command{ + "list": blockListCmd, "stat": blockStatCmd, "get": blockGetCmd, }, diff --git a/core/commands/shutdown.go b/core/commands/shutdown.go index 8e6539a..86e7ce6 100644 --- a/core/commands/shutdown.go +++ b/core/commands/shutdown.go @@ -3,7 +3,7 @@ package commands import ( "fmt" - cmds "github.com/Casper-dev/Casper-server/commands" + cmds "gitlab.com/casperDev/Casper-server/commands" ) var daemonShutdownCmd = &cmds.Command{ diff --git a/core/commands/stat.go b/core/commands/stat.go index 625cbf1..ca0f105 100644 --- a/core/commands/stat.go +++ b/core/commands/stat.go @@ -9,11 +9,12 @@ import ( humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" - cmds "github.com/Casper-dev/Casper-server/commands" metrics "gx/ipfs/QmQbh3Rb7KM37As3vkHYnEFnzkVXNCP8EYGtHz6g2fXk14/go-libp2p-metrics" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + + cmds "gitlab.com/casperDev/Casper-server/commands" ) var StatsCmd = &cmds.Command{ diff --git a/core/commands/swarm.go b/core/commands/swarm.go index d715ca8..22e3905 100644 --- a/core/commands/swarm.go +++ b/core/commands/swarm.go @@ -9,14 +9,15 @@ import ( "sort" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - repo "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - "github.com/Casper-dev/Casper-server/repo/fsrepo" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" swarm "gx/ipfs/QmdQFrFnPrKRQtpeHKjZ3cVNwxmGKKS2TvhJTuN9C9yduh/go-libp2p-swarm" iaddr "gx/ipfs/QmeS8cCKawUwejVrsBtmC1toTXmwVWZGiRJqzgTURVWeF9/go-ipfs-addr" + cmds "gitlab.com/casperDev/Casper-server/commands" + repo "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + "gitlab.com/casperDev/Casper-server/repo/fsrepo" + mafilter "gx/ipfs/QmSMZwvs3n4GBikZ7hKzT17c3bk65FmyZo2JqtJ16swqCv/multiaddr-filter" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" ) diff --git a/core/commands/sysdiag.go b/core/commands/sysdiag.go index c3e78e2..785305e 100644 --- a/core/commands/sysdiag.go +++ b/core/commands/sysdiag.go @@ -5,8 +5,8 @@ import ( "path" "runtime" - cmds "github.com/Casper-dev/Casper-server/commands" - config "github.com/Casper-dev/Casper-server/repo/config" + cmds "gitlab.com/casperDev/Casper-server/commands" + config "gitlab.com/casperDev/Casper-server/repo/config" manet "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" sysi "gx/ipfs/QmZRjKbHa6DenStpQJFiaPcEwkZqrx7TH6xTf342LDU3qM/go-sysinfo" @@ -79,14 +79,14 @@ func runtimeInfo(out map[string]interface{}) error { func envVarInfo(out map[string]interface{}) error { ev := make(map[string]interface{}) ev["GOPATH"] = os.Getenv("GOPATH") - ev["IPFS_PATH"] = os.Getenv("CASPER_SERVER_PATH") + ev["IPFS_PATH"] = os.Getenv(config.EnvDir) out["environment"] = ev return nil } func ipfsPath() string { - p := os.Getenv("CASPER_SERVER_PATH") + p := os.Getenv(config.EnvDir) if p == "" { p = path.Join(os.Getenv("HOME"), ".ipfs") } diff --git a/core/commands/tar.go b/core/commands/tar.go index acb7392..76af748 100644 --- a/core/commands/tar.go +++ b/core/commands/tar.go @@ -4,12 +4,12 @@ import ( "io" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/core/coreunix" - dag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" - tar "github.com/Casper-dev/Casper-server/tar" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core/coreunix" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + tar "gitlab.com/casperDev/Casper-server/tar" ) var TarCmd = &cmds.Command{ diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index f3dcba9..9072460 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -7,13 +7,13 @@ import ( "sort" "text/tabwriter" - cmds "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - merkledag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" - unixfs "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" - unixfspb "github.com/Casper-dev/Casper-server/unixfs/pb" + cmds "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + merkledag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + unixfs "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + unixfspb "gitlab.com/casperDev/Casper-server/unixfs/pb" ) type LsLink struct { diff --git a/core/commands/unixfs/unixfs.go b/core/commands/unixfs/unixfs.go index 13d31da..9dc0571 100644 --- a/core/commands/unixfs/unixfs.go +++ b/core/commands/unixfs/unixfs.go @@ -1,6 +1,6 @@ package unixfs -import cmds "github.com/Casper-dev/Casper-server/commands" +import cmds "gitlab.com/casperDev/Casper-server/commands" var UnixFSCmd = &cmds.Command{ Helptext: cmds.HelpText{ diff --git a/core/commands/upd.go b/core/commands/upd.go new file mode 100644 index 0000000..8950827 --- /dev/null +++ b/core/commands/upd.go @@ -0,0 +1,99 @@ +package commands + +import ( + "fmt" + util "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" + "io" + "strings" + + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + "gitlab.com/casperDev/Casper-server/blockservice" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-server/exchange/offline" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + pin "gitlab.com/casperDev/Casper-server/pin" +) + +var UpdCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Updates file corresponding to specified UUID.", + ShortDescription: ` +Recursively downloads blocks and pins them to local storage. +`, + LongDescription: ` +Makes DAG node with UUID contain . + is recursively downloaded and pinned to local storage. +`, + }, + + Arguments: []cmds.Argument{ + cmds.StringArg("uuid", true, false, "Base58 encoded UUID."), + cmds.StringArg("ipfs-path", true, false, "The path to the IPFS object."), + }, + Run: func(req cmds.Request, res cmds.Response) { + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + p, err := path.ParsePath(req.Arguments()[1]) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + obj, _, err := n.Resolver.ResolveToLastNode(req.Context(), p) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + pn := obj.(*dag.ProtoNode) + + addblockstore := blockstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker) + exch := offline.Exchange(addblockstore) + bserv := blockservice.New(addblockstore, exch) + dserv := dag.NewDAGService(bserv) + + defer n.Blockstore.PinLock().Unlock() + + // TODO: make an option to disable this behaviour + oldCid := pn.Cid() + log.Debugf("Remove pin on CID: %s", oldCid.String()) + n.Pinning.RemovePinWithMode(oldCid, pin.Recursive) + + pn.SetUUID(base58.Decode(req.Arguments()[0])) + + rnk, err := dserv.Add(pn) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + log.Debugf("Pin CID: %s", rnk.String()) + n.Pinning.PinWithMode(rnk, pin.Recursive) + + if err = n.Pinning.Flush(); err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + res.SetOutput(pn) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + pn, ok := res.Output().(*dag.ProtoNode) + if !ok { + return nil, util.ErrCast() + } + + return strings.NewReader(fmt.Sprintf("UUID: %s\nHash: %s\n", + base58.Encode(pn.UUID()), + pn.Cid().String(), + )), nil + }, + }, + Type: dag.ProtoNode{}, +} diff --git a/core/commands/validate.go b/core/commands/validate.go new file mode 100644 index 0000000..59c50ea --- /dev/null +++ b/core/commands/validate.go @@ -0,0 +1,77 @@ +package commands + +import ( + cu "gitlab.com/casperDev/Casper-server/casper/casper_utils" + thrift "gitlab.com/casperDev/Casper-server/casper/thrift" + val "gitlab.com/casperDev/Casper-server/casper/validation" + cmds "gitlab.com/casperDev/Casper-server/commands" + "gitlab.com/casperDev/Casper-thrift/casperproto" + + "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" + "gx/ipfs/QmeS8cCKawUwejVrsBtmC1toTXmwVWZGiRJqzgTURVWeF9/go-ipfs-addr" +) + +var ValidateCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Validate specific UUID.", + ShortDescription: ` +Lists running and recently run commands. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("uuid", true, false, "UUID to validate"), + }, + Options: []cmds.Option{ + cmds.StringOption("server", "s", "Perform validation as client"), + cmds.StringOption("node", "Node ID"), + }, + Run: func(req cmds.Request, res cmds.Response) { + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + id := req.Arguments()[0] + server, sf, _ := req.Option("server").String() + if sf { + localAddr := cu.GetLocalAddr() + n, nf, _ := req.Option("node").String() + if nf { + a, err := ipfsaddr.ParseString(n) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + taddr, err := manet.ToNetAddr(a.Transport()) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + localAddr = &cu.ExternalAddr{a, taddr} + } + log.Debugf("Address: %s", localAddr.String()) + + _, err = thrift.RunClientClosure(server, func(c *thrift.ThriftClient) (interface{}, error) { + return nil, c.SendVerificationQuery(req.Context(), id, &casperproto.NodeInfo{ + IpfsAddr: localAddr.IPFS().String(), + ThriftAddr: localAddr.Thrift().String(), + }) + }) + if err != nil { + log.Error(err) + res.SetError(err, cmds.ErrNormal) + return + } + } else { + err := val.PerformValidation(req.Context(), n, id) + if err != nil { + log.Error(err) + res.SetError(err, cmds.ErrNormal) + return + } + } + res.SetOutput(nil) + }, +} diff --git a/core/commands/version.go b/core/commands/version.go index e5f5712..4e6d484 100644 --- a/core/commands/version.go +++ b/core/commands/version.go @@ -6,9 +6,9 @@ import ( "runtime" "strings" - cmds "github.com/Casper-dev/Casper-server/commands" - config "github.com/Casper-dev/Casper-server/repo/config" - fsrepo "github.com/Casper-dev/Casper-server/repo/fsrepo" + cmds "gitlab.com/casperDev/Casper-server/commands" + config "gitlab.com/casperDev/Casper-server/repo/config" + fsrepo "gitlab.com/casperDev/Casper-server/repo/fsrepo" ) type VersionOutput struct { diff --git a/core/core.go b/core/core.go index 4112678..83dfd40 100644 --- a/core/core.go +++ b/core/core.go @@ -12,6 +12,7 @@ package core import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -19,28 +20,31 @@ import ( "net" "os" "strings" + "sync" "time" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - bserv "github.com/Casper-dev/Casper-server/blockservice" - exchange "github.com/Casper-dev/Casper-server/exchange" - bitswap "github.com/Casper-dev/Casper-server/exchange/bitswap" - bsnet "github.com/Casper-dev/Casper-server/exchange/bitswap/network" - rp "github.com/Casper-dev/Casper-server/exchange/reprovide" - filestore "github.com/Casper-dev/Casper-server/filestore" - mount "github.com/Casper-dev/Casper-server/fuse/mount" - merkledag "github.com/Casper-dev/Casper-server/merkledag" - mfs "github.com/Casper-dev/Casper-server/mfs" - namesys "github.com/Casper-dev/Casper-server/namesys" - ipnsrp "github.com/Casper-dev/Casper-server/namesys/republisher" - p2p "github.com/Casper-dev/Casper-server/p2p" - path "github.com/Casper-dev/Casper-server/path" - pin "github.com/Casper-dev/Casper-server/pin" - repo "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - nilrouting "github.com/Casper-dev/Casper-server/routing/none" - offroute "github.com/Casper-dev/Casper-server/routing/offline" - ft "github.com/Casper-dev/Casper-server/unixfs" + bl "gitlab.com/casperDev/Casper-server/blocks" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bserv "gitlab.com/casperDev/Casper-server/blockservice" + uid "gitlab.com/casperDev/Casper-server/casper/uuid" + exchange "gitlab.com/casperDev/Casper-server/exchange" + bitswap "gitlab.com/casperDev/Casper-server/exchange/bitswap" + bsnet "gitlab.com/casperDev/Casper-server/exchange/bitswap/network" + rp "gitlab.com/casperDev/Casper-server/exchange/reprovide" + filestore "gitlab.com/casperDev/Casper-server/filestore" + mount "gitlab.com/casperDev/Casper-server/fuse/mount" + merkledag "gitlab.com/casperDev/Casper-server/merkledag" + mfs "gitlab.com/casperDev/Casper-server/mfs" + namesys "gitlab.com/casperDev/Casper-server/namesys" + ipnsrp "gitlab.com/casperDev/Casper-server/namesys/republisher" + p2p "gitlab.com/casperDev/Casper-server/p2p" + path "gitlab.com/casperDev/Casper-server/path" + pin "gitlab.com/casperDev/Casper-server/pin" + repo "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + nilrouting "gitlab.com/casperDev/Casper-server/routing/none" + offroute "gitlab.com/casperDev/Casper-server/routing/offline" + ft "gitlab.com/casperDev/Casper-server/unixfs" yamux "gx/ipfs/QmNWCEvi7bPRcvqAV8AKLGVNoQdArWi7NJayka2SM4XtRe/go-smux-yamux" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" @@ -58,6 +62,7 @@ import ( floodsub "gx/ipfs/QmUUSLfvihARhCxxgnjW4hmycJpPvzNu12Aaz6JWVdfnLg/go-libp2p-floodsub" addrutil "gx/ipfs/QmVJGsPeK3vwtEyyTxpCs47yjBYMmYsAhEouPDF3Gb2eK3/go-addr-util" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" + query "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/query" mssmux "gx/ipfs/QmVniQJkdzLZaZwzwMdd3dJTvWiJ1DQEkreVy6hs6h7Vk5/go-smux-multistream" dht "gx/ipfs/QmWRBYr99v8sjrpbyNWMuGkQekn7b9ELoLSCe8Ny7Nxain/go-libp2p-kad-dht" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" @@ -69,6 +74,7 @@ import ( p2phost "gx/ipfs/Qmc1XhrFEiSeBNn3mpfg6gEuYCt5im2gYmNVmncsvmpeAk/go-libp2p-host" pnet "gx/ipfs/QmcWmYQEQCrezztaQ81nXzMx2jaAEow17wdesDAjjR769r/go-libp2p-pnet" swarm "gx/ipfs/QmdQFrFnPrKRQtpeHKjZ3cVNwxmGKKS2TvhJTuN9C9yduh/go-libp2p-swarm" + iaddr "gx/ipfs/QmeS8cCKawUwejVrsBtmC1toTXmwVWZGiRJqzgTURVWeF9/go-ipfs-addr" discovery "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/discovery" p2pbhost "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/host/basic" rhost "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/host/routed" @@ -709,10 +715,131 @@ func (n *IpfsNode) loadBootstrapPeers() ([]pstore.PeerInfo, error) { return toPeerInfos(parsed), nil } +const ( + uuidDSKeyPrefix = "/local/uuid/" + uuidChanBufLen = 128 +) + +type UUIDInfo struct { + PubKey string +} + +var UUIDInfoCache = &sync.Map{} + +func keyByUUID(uuid string) ds.Key { + return ds.NewKey(uuidDSKeyPrefix + uuid) +} + +func (n *IpfsNode) InitUUIDCache(ctx context.Context) error { + q := query.Query{KeysOnly: true, Prefix: uuidDSKeyPrefix} + res, err := n.Repo.Datastore().Query(q) + if err != nil { + return err + } + + defer res.Close() + for { + e, ok := res.NextSync() + if !ok { + return nil + } + if e.Error != nil { + return e.Error + } + res := strings.Split(e.Key, "/") + select { + case <-ctx.Done(): + return ctx.Err() + default: + info := UUIDInfo{} + err := json.Unmarshal(e.Value.([]byte), &info) + if err != nil { + // no return here because we anyway try to get UUID from DS + // if it does not exist in cache + log.Error(err) + } + UUIDInfoCache.Store(res[3], &info) + } + } + + return nil +} + +func (n *IpfsNode) AddUUID(uuid string, value *UUIDInfo) error { + key := keyByUUID(uuid) + log.Debugf("adding UUID '%s' to local datastore", uuid) + + b, err := json.Marshal(value) + if err != nil { + return err + } + + UUIDInfoCache.Store(uuid, value) + return n.Repo.Datastore().Put(key, b) +} + +func (n *IpfsNode) RemoveUUID(uuid string) error { + key := keyByUUID(uuid) + log.Debugf("removing UUID '%s' from local datastore", uuid) + UUIDInfoCache.Delete(uuid) + return n.Repo.Datastore().Delete(key) +} + +func (n *IpfsNode) GetUUID(uuid string) (*UUIDInfo, error) { + key := keyByUUID(uuid) + val, err := n.Repo.Datastore().Get(key) + if err != nil { + return nil, err + } + + if val, ok := UUIDInfoCache.Load(uuid); ok { + return val.(*UUIDInfo), nil + } + + info := &UUIDInfo{} + if json.Unmarshal(val.([]byte), &info) != nil { + return nil, err + } + return info, err +} + +func (n *IpfsNode) AllUUIDChan(ctx context.Context) (<-chan string, error) { + q := query.Query{KeysOnly: true, Prefix: uuidDSKeyPrefix} + res, err := n.Repo.Datastore().Query(q) + if err != nil { + return nil, err + } + + output := make(chan string, uuidChanBufLen) + go func() { + defer close(output) + defer res.Close() + + for { + e, ok := res.NextSync() + if !ok { + return + } + if e.Error != nil { + log.Errorf("error while getting UUID channel: %v", e.Error) + return + } + res := strings.Split(e.Key, "/") + select { + case <-ctx.Done(): + return + case output <- res[3]: + } + } + }() + + return output, nil +} + func (n *IpfsNode) loadFilesRoot() error { dsk := ds.NewKey("/local/filesroot") pf := func(ctx context.Context, c *cid.Cid) error { - return n.Repo.Datastore().Put(dsk, c.Bytes()) + return n.Repo.Datastore().Put(dsk, append(uid.NullUUID, c.Bytes()...)) } var nd *merkledag.ProtoNode @@ -721,12 +848,16 @@ func (n *IpfsNode) loadFilesRoot() error { switch { case err == ds.ErrNotFound || val == nil: nd = ft.EmptyDirNode() + fmt.Printf("EmptyDirNode: %+v\n", nd) _, err := n.DAG.Add(nd) if err != nil { return fmt.Errorf("failure writing to dagstore: %s", err) } case err == nil: - c, err := cid.Cast(val.([]byte)) + _, v := bl.SplitData(val.([]byte)) + + c, err := cid.Cast(v) + if err != nil { return err } @@ -811,6 +942,45 @@ func listenAddresses(cfg *config.Config) ([]ma.Multiaddr, error) { return listen, nil } +func (n *IpfsNode) ConnectToPeer(ctx context.Context, saddr string) error { + if n.PeerHost == nil { + return errors.New("Can connect to peers only on daemon") + } + + snet, ok := n.PeerHost.Network().(*swarm.Network) + if !ok { + return errors.New("Peerhost network is not swarm") + } + + ia, err := iaddr.ParseString(saddr) + if err != nil { + return err + } + swrm := snet.Swarm() + pi := pstore.PeerInfo{ + ID: ia.ID(), + Addrs: []ma.Multiaddr{ia.Transport()}, + } + + swrm.Backoff().Clear(pi.ID) + return n.PeerHost.Connect(ctx, pi) +} + +func (n *IpfsNode) ConnectToPeers(ctx context.Context, addrs ...string) *sync.WaitGroup { + wg := &sync.WaitGroup{} + for _, addr := range addrs { + wg.Add(1) + go func(addr string) { + defer wg.Done() + err := n.ConnectToPeer(ctx, addr) + if err != nil { + log.Error(err) + } + }(addr) + } + return wg +} + type ConstructPeerHostOpts struct { AddrsFactory p2pbhost.AddrsFactory DisableNatPortMap bool diff --git a/core/core_test.go b/core/core_test.go index c3de5b2..3f95664 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -4,9 +4,10 @@ import ( "testing" context "context" - "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - ds2 "github.com/Casper-dev/Casper-server/thirdparty/datastore2" + + "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + ds2 "gitlab.com/casperDev/Casper-server/thirdparty/datastore2" ) func TestInitialization(t *testing.T) { diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index 95ee34b..ff08d4c 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -3,10 +3,10 @@ package coreapi import ( "context" - core "github.com/Casper-dev/Casper-server/core" - coreiface "github.com/Casper-dev/Casper-server/core/coreapi/interface" - ipfspath "github.com/Casper-dev/Casper-server/path" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + core "gitlab.com/casperDev/Casper-server/core" + coreiface "gitlab.com/casperDev/Casper-server/core/coreapi/interface" + ipfspath "gitlab.com/casperDev/Casper-server/path" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ) diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 76c2f01..2377c4d 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -4,9 +4,9 @@ import ( "context" "io" - coreiface "github.com/Casper-dev/Casper-server/core/coreapi/interface" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + coreiface "gitlab.com/casperDev/Casper-server/core/coreapi/interface" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/coreapi/unixfs_test.go b/core/coreapi/unixfs_test.go index 933e467..239e2b4 100644 --- a/core/coreapi/unixfs_test.go +++ b/core/coreapi/unixfs_test.go @@ -8,16 +8,17 @@ import ( "strings" "testing" - core "github.com/Casper-dev/Casper-server/core" - coreapi "github.com/Casper-dev/Casper-server/core/coreapi" - coreiface "github.com/Casper-dev/Casper-server/core/coreapi/interface" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" - mdag "github.com/Casper-dev/Casper-server/merkledag" - repo "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - ds2 "github.com/Casper-dev/Casper-server/thirdparty/datastore2" - unixfs "github.com/Casper-dev/Casper-server/unixfs" cbor "gx/ipfs/QmWCs8kMecJwCPK8JThue8TjgM2ieJ2HjTLDu7Cv2NEmZi/go-ipld-cbor" + + core "gitlab.com/casperDev/Casper-server/core" + coreapi "gitlab.com/casperDev/Casper-server/core/coreapi" + coreiface "gitlab.com/casperDev/Casper-server/core/coreapi/interface" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + repo "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + ds2 "gitlab.com/casperDev/Casper-server/thirdparty/datastore2" + unixfs "gitlab.com/casperDev/Casper-server/unixfs" ) // `echo -n 'hello, world!' | ipfs add` diff --git a/core/coredag/dagpb.go b/core/coredag/dagpb.go index ff00259..fb57328 100644 --- a/core/coredag/dagpb.go +++ b/core/coredag/dagpb.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "math" - "github.com/Casper-dev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/merkledag" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/coredag/raw.go b/core/coredag/raw.go index eca6c55..aaa5436 100644 --- a/core/coredag/raw.go +++ b/core/coredag/raw.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "math" - "github.com/Casper-dev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/merkledag" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/corehttp/commands.go b/core/corehttp/commands.go index 2b72582..76e8015 100644 --- a/core/corehttp/commands.go +++ b/core/corehttp/commands.go @@ -7,11 +7,11 @@ import ( "strconv" "strings" - commands "github.com/Casper-dev/Casper-server/commands" - cmdsHttp "github.com/Casper-dev/Casper-server/commands/http" - core "github.com/Casper-dev/Casper-server/core" - corecommands "github.com/Casper-dev/Casper-server/core/commands" - config "github.com/Casper-dev/Casper-server/repo/config" + commands "gitlab.com/casperDev/Casper-server/commands" + cmdsHttp "gitlab.com/casperDev/Casper-server/commands/http" + core "gitlab.com/casperDev/Casper-server/core" + corecommands "gitlab.com/casperDev/Casper-server/core/commands" + config "gitlab.com/casperDev/Casper-server/repo/config" ) const originEnvKey = "API_ORIGIN" diff --git a/core/corehttp/corehttp.go b/core/corehttp/corehttp.go index 71e24d5..bf450f0 100644 --- a/core/corehttp/corehttp.go +++ b/core/corehttp/corehttp.go @@ -10,11 +10,12 @@ import ( "net/http" "time" - core "github.com/Casper-dev/Casper-server/core" "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" manet "gx/ipfs/QmX3U3YXCQ6UYBxq2LVWF8dARS1hPUTEYLrSx654Qyxyw6/go-multiaddr-net" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" + + core "gitlab.com/casperDev/Casper-server/core" ) var log = logging.Logger("core/server") diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index e990c26..f014e58 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -5,10 +5,11 @@ import ( "net" "net/http" - core "github.com/Casper-dev/Casper-server/core" - coreapi "github.com/Casper-dev/Casper-server/core/coreapi" - config "github.com/Casper-dev/Casper-server/repo/config" id "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/protocol/identify" + + core "gitlab.com/casperDev/Casper-server/core" + coreapi "gitlab.com/casperDev/Casper-server/core/coreapi" + config "gitlab.com/casperDev/Casper-server/repo/config" ) type GatewayConfig struct { diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index aac6871..80aa743 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -12,16 +12,16 @@ import ( "strings" "time" - core "github.com/Casper-dev/Casper-server/core" - coreapi "github.com/Casper-dev/Casper-server/core/coreapi" - coreiface "github.com/Casper-dev/Casper-server/core/coreapi/interface" - "github.com/Casper-dev/Casper-server/importer" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - dag "github.com/Casper-dev/Casper-server/merkledag" - dagutils "github.com/Casper-dev/Casper-server/merkledag/utils" - path "github.com/Casper-dev/Casper-server/path" - ft "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + core "gitlab.com/casperDev/Casper-server/core" + coreapi "gitlab.com/casperDev/Casper-server/core/coreapi" + coreiface "gitlab.com/casperDev/Casper-server/core/coreapi/interface" + "gitlab.com/casperDev/Casper-server/importer" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + dag "gitlab.com/casperDev/Casper-server/merkledag" + dagutils "gitlab.com/casperDev/Casper-server/merkledag/utils" + path "gitlab.com/casperDev/Casper-server/path" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/corehttp/gateway_indexPage.go b/core/corehttp/gateway_indexPage.go index 1cce1e5..968986e 100644 --- a/core/corehttp/gateway_indexPage.go +++ b/core/corehttp/gateway_indexPage.go @@ -6,7 +6,7 @@ import ( "path" "strings" - "github.com/Casper-dev/Casper-server/assets" + "gitlab.com/casperDev/Casper-server/assets" ) // structs for directory listing diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index 9e57876..c409193 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -10,14 +10,14 @@ import ( "testing" "time" - core "github.com/Casper-dev/Casper-server/core" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" - dag "github.com/Casper-dev/Casper-server/merkledag" - namesys "github.com/Casper-dev/Casper-server/namesys" - path "github.com/Casper-dev/Casper-server/path" - repo "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - ds2 "github.com/Casper-dev/Casper-server/thirdparty/datastore2" + core "gitlab.com/casperDev/Casper-server/core" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + dag "gitlab.com/casperDev/Casper-server/merkledag" + namesys "gitlab.com/casperDev/Casper-server/namesys" + path "gitlab.com/casperDev/Casper-server/path" + repo "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + ds2 "gitlab.com/casperDev/Casper-server/thirdparty/datastore2" ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" id "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/protocol/identify" diff --git a/core/corehttp/ipns_hostname.go b/core/corehttp/ipns_hostname.go index 0aa5c54..94ed10f 100644 --- a/core/corehttp/ipns_hostname.go +++ b/core/corehttp/ipns_hostname.go @@ -6,7 +6,7 @@ import ( "net/http" "strings" - "github.com/Casper-dev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core" isd "gx/ipfs/QmZmmuAXgX73UQmX1jRKjTGmjzq24Jinqkq8vzkBtno4uX/go-is-domain" ) diff --git a/core/corehttp/logs.go b/core/corehttp/logs.go index bf34c60..658dc70 100644 --- a/core/corehttp/logs.go +++ b/core/corehttp/logs.go @@ -5,8 +5,9 @@ import ( "net" "net/http" - core "github.com/Casper-dev/Casper-server/core" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + + core "gitlab.com/casperDev/Casper-server/core" ) type writeErrNotifier struct { diff --git a/core/corehttp/metrics.go b/core/corehttp/metrics.go index 71ab4e3..a5cb404 100644 --- a/core/corehttp/metrics.go +++ b/core/corehttp/metrics.go @@ -4,7 +4,7 @@ import ( "net" "net/http" - core "github.com/Casper-dev/Casper-server/core" + core "gitlab.com/casperDev/Casper-server/core" prometheus "gx/ipfs/QmX3QZ5jHEPidwUrymXV1iSCSUhdGxj15sm2gP4jKMef7B/client_golang/prometheus" ) diff --git a/core/corehttp/metrics_test.go b/core/corehttp/metrics_test.go index d76e330..a3b6016 100644 --- a/core/corehttp/metrics_test.go +++ b/core/corehttp/metrics_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - core "github.com/Casper-dev/Casper-server/core" + core "gitlab.com/casperDev/Casper-server/core" inet "gx/ipfs/QmNa31VPzC561NWwRsJLE7nGYZYuuD2QfpK2b1q9BK54J1/go-libp2p-net" testutil "gx/ipfs/QmQGX417WoxKxDJeHqouMEmmH4G1RCENNSzkZYHrXy3Xb3/go-libp2p-netutil" diff --git a/core/corehttp/redirect.go b/core/corehttp/redirect.go index 797de08..53c67c0 100644 --- a/core/corehttp/redirect.go +++ b/core/corehttp/redirect.go @@ -4,7 +4,7 @@ import ( "net" "net/http" - core "github.com/Casper-dev/Casper-server/core" + core "gitlab.com/casperDev/Casper-server/core" ) func RedirectOption(path string, redirect string) ServeOption { diff --git a/core/corerepo/gc.go b/core/corerepo/gc.go index 357eb6a..338f307 100644 --- a/core/corerepo/gc.go +++ b/core/corerepo/gc.go @@ -6,10 +6,10 @@ import ( "errors" "time" - "github.com/Casper-dev/Casper-server/core" - mfs "github.com/Casper-dev/Casper-server/mfs" - gc "github.com/Casper-dev/Casper-server/pin/gc" - repo "github.com/Casper-dev/Casper-server/repo" + "gitlab.com/casperDev/Casper-server/core" + mfs "gitlab.com/casperDev/Casper-server/mfs" + gc "gitlab.com/casperDev/Casper-server/pin/gc" + repo "gitlab.com/casperDev/Casper-server/repo" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" diff --git a/core/corerepo/pinning.go b/core/corerepo/pinning.go index 18cdfb9..7176f75 100644 --- a/core/corerepo/pinning.go +++ b/core/corerepo/pinning.go @@ -17,9 +17,9 @@ import ( "context" "fmt" - "github.com/Casper-dev/Casper-server/core" - path "github.com/Casper-dev/Casper-server/path" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + "gitlab.com/casperDev/Casper-server/core" + path "gitlab.com/casperDev/Casper-server/path" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/corerepo/stat.go b/core/corerepo/stat.go index 75675f4..b4b19fd 100644 --- a/core/corerepo/stat.go +++ b/core/corerepo/stat.go @@ -5,8 +5,9 @@ import ( "math" context "context" - "github.com/Casper-dev/Casper-server/core" - fsrepo "github.com/Casper-dev/Casper-server/repo/fsrepo" + + "gitlab.com/casperDev/Casper-server/core" + fsrepo "gitlab.com/casperDev/Casper-server/repo/fsrepo" humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" ) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index bd83e7c..bdc1c27 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -9,25 +9,28 @@ import ( gopath "path" "strconv" - bs "github.com/Casper-dev/Casper-server/blocks/blockstore" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - bserv "github.com/Casper-dev/Casper-server/blockservice" - "github.com/Casper-dev/Casper-server/commands/files" - core "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/exchange/offline" - balanced "github.com/Casper-dev/Casper-server/importer/balanced" - "github.com/Casper-dev/Casper-server/importer/chunk" - ihelper "github.com/Casper-dev/Casper-server/importer/helpers" - trickle "github.com/Casper-dev/Casper-server/importer/trickle" - dag "github.com/Casper-dev/Casper-server/merkledag" - mfs "github.com/Casper-dev/Casper-server/mfs" - "github.com/Casper-dev/Casper-server/pin" - posinfo "github.com/Casper-dev/Casper-server/thirdparty/posinfo" - unixfs "github.com/Casper-dev/Casper-server/unixfs" + bl "gitlab.com/casperDev/Casper-server/blocks" + bs "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bserv "gitlab.com/casperDev/Casper-server/blockservice" + "gitlab.com/casperDev/Casper-server/commands/files" + core "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/exchange/offline" + balanced "gitlab.com/casperDev/Casper-server/importer/balanced" + "gitlab.com/casperDev/Casper-server/importer/chunk" + ihelper "gitlab.com/casperDev/Casper-server/importer/helpers" + trickle "gitlab.com/casperDev/Casper-server/importer/trickle" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mfs "gitlab.com/casperDev/Casper-server/mfs" + "gitlab.com/casperDev/Casper-server/pin" + posinfo "gitlab.com/casperDev/Casper-server/thirdparty/posinfo" + unixfs "gitlab.com/casperDev/Casper-server/unixfs" + //uuid "gitlab.com/casperDev/Casper-server/casper/uuid" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" syncds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" ) @@ -70,6 +73,7 @@ type AddedObject struct { Name string Hash string `json:",omitempty"` Bytes int64 `json:",omitempty"` + UUID string `json:",omitempty"` Size string `json:",omitempty"` } @@ -182,6 +186,10 @@ func (adder *Adder) RootNode() (node.Node, error) { } func (adder *Adder) PinRoot() error { + return adder.PinRootUUID(nil) +} + +func (adder *Adder) PinRootUUID(uuid []byte) error { root, err := adder.RootNode() if err != nil { return err @@ -190,6 +198,11 @@ func (adder *Adder) PinRoot() error { return nil } + if uuid != nil { + pn := root.(*dag.ProtoNode) + pn.SetUUID(uuid) + } + rnk, err := adder.dagService.Add(root) if err != nil { return err @@ -329,7 +342,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { return "", err } - f, err := files.NewSerialFile(root, root, false, stat) + f, err := files.NewSerialFile(root, root, false, stat, nil) if err != nil { return "", err } @@ -485,6 +498,7 @@ func (adder *Adder) addFile(file files.File) error { } dagnode, err := adder.add(reader) + if err != nil { return err } @@ -556,9 +570,11 @@ func outputDagnode(out chan interface{}, name string, dn node.Node) error { return err } + u, _ := bl.SplitData(dn.RawData()) out <- &AddedObject{ Hash: o.Hash, Name: name, + UUID: base58.Encode(u), Size: o.Size, } diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 1215a34..285a525 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -10,18 +10,19 @@ import ( "testing" "time" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - "github.com/Casper-dev/Casper-server/blockservice" - "github.com/Casper-dev/Casper-server/commands/files" - "github.com/Casper-dev/Casper-server/core" - dag "github.com/Casper-dev/Casper-server/merkledag" - "github.com/Casper-dev/Casper-server/pin/gc" - "github.com/Casper-dev/Casper-server/repo" - "github.com/Casper-dev/Casper-server/repo/config" - ds2 "github.com/Casper-dev/Casper-server/thirdparty/datastore2" - pi "github.com/Casper-dev/Casper-server/thirdparty/posinfo" "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + "gitlab.com/casperDev/Casper-server/blockservice" + "gitlab.com/casperDev/Casper-server/commands/files" + "gitlab.com/casperDev/Casper-server/core" + dag "gitlab.com/casperDev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/pin/gc" + "gitlab.com/casperDev/Casper-server/repo" + "gitlab.com/casperDev/Casper-server/repo/config" + ds2 "gitlab.com/casperDev/Casper-server/thirdparty/datastore2" + pi "gitlab.com/casperDev/Casper-server/thirdparty/posinfo" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ) diff --git a/core/coreunix/cat.go b/core/coreunix/cat.go index 8a03be7..e5f09c1 100644 --- a/core/coreunix/cat.go +++ b/core/coreunix/cat.go @@ -3,9 +3,13 @@ package coreunix import ( "context" - core "github.com/Casper-dev/Casper-server/core" - path "github.com/Casper-dev/Casper-server/path" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + "fmt" + + core "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" ) func Cat(ctx context.Context, n *core.IpfsNode, pstr string) (uio.DagReader, error) { @@ -18,6 +22,16 @@ func Cat(ctx context.Context, n *core.IpfsNode, pstr string) (uio.DagReader, err if err != nil { return nil, err } + fmt.Println(dagNode) + fmt.Println(err) + + if v, ok := dagNode.(*merkledag.ProtoNode); ok { + if fsn, err := ft.FSNodeFromBytes(v.Data()); err == nil { + if fsn.Type == ft.TDirectory && len(fsn.Data) == 0 && len(v.Links()) == 1 { + dagNode, _ = v.Links()[0].GetNode(ctx, n.DAG) + } + } + } return uio.NewDagReader(ctx, dagNode, n.DAG) } diff --git a/core/coreunix/metadata.go b/core/coreunix/metadata.go index eac853f..365a97e 100644 --- a/core/coreunix/metadata.go +++ b/core/coreunix/metadata.go @@ -1,10 +1,11 @@ package coreunix import ( - core "github.com/Casper-dev/Casper-server/core" - dag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + + core "gitlab.com/casperDev/Casper-server/core" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" ) func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error) { diff --git a/core/coreunix/metadata_test.go b/core/coreunix/metadata_test.go index 8dfb413..3a2f19a 100644 --- a/core/coreunix/metadata_test.go +++ b/core/coreunix/metadata_test.go @@ -5,15 +5,15 @@ import ( "io/ioutil" "testing" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - bserv "github.com/Casper-dev/Casper-server/blockservice" - core "github.com/Casper-dev/Casper-server/core" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - importer "github.com/Casper-dev/Casper-server/importer" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - merkledag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bserv "gitlab.com/casperDev/Casper-server/blockservice" + core "gitlab.com/casperDev/Casper-server/core" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + importer "gitlab.com/casperDev/Casper-server/importer" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + merkledag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" context "context" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" diff --git a/core/mock/mock.go b/core/mock/mock.go index 5fccb5c..ae714d8 100644 --- a/core/mock/mock.go +++ b/core/mock/mock.go @@ -4,13 +4,14 @@ import ( "context" "net" - commands "github.com/Casper-dev/Casper-server/commands" - core "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/repo" - config "github.com/Casper-dev/Casper-server/repo/config" - ds2 "github.com/Casper-dev/Casper-server/thirdparty/datastore2" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + commands "gitlab.com/casperDev/Casper-server/commands" + core "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/repo" + config "gitlab.com/casperDev/Casper-server/repo/config" + ds2 "gitlab.com/casperDev/Casper-server/thirdparty/datastore2" + pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" metrics "gx/ipfs/QmQbh3Rb7KM37As3vkHYnEFnzkVXNCP8EYGtHz6g2fXk14/go-libp2p-metrics" ipnet "gx/ipfs/QmQq9YzmdFdWNTDdArueGyD7L5yyiRQigrRHJnTGkxcEjT/go-libp2p-interface-pnet" diff --git a/core/pathresolver.go b/core/pathresolver.go index 53185b0..eff7a6d 100644 --- a/core/pathresolver.go +++ b/core/pathresolver.go @@ -5,8 +5,8 @@ import ( "errors" "strings" - namesys "github.com/Casper-dev/Casper-server/namesys" - path "github.com/Casper-dev/Casper-server/path" + namesys "gitlab.com/casperDev/Casper-server/namesys" + path "gitlab.com/casperDev/Casper-server/path" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/core/pathresolver_test.go b/core/pathresolver_test.go index 91b493f..911bf2e 100644 --- a/core/pathresolver_test.go +++ b/core/pathresolver_test.go @@ -3,9 +3,9 @@ package core_test import ( "testing" - core "github.com/Casper-dev/Casper-server/core" - coremock "github.com/Casper-dev/Casper-server/core/mock" - path "github.com/Casper-dev/Casper-server/path" + core "gitlab.com/casperDev/Casper-server/core" + coremock "gitlab.com/casperDev/Casper-server/core/mock" + path "gitlab.com/casperDev/Casper-server/path" ) func TestResolveNoComponents(t *testing.T) { diff --git a/doc.go b/doc.go deleted file mode 100644 index fdf2f69..0000000 --- a/doc.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -IPFS is a global, versioned, peer-to-peer filesystem - -There are sub-packages within the ipfs package for various low-level -utilities, which are in turn assembled into: - -core/...: - The low-level API that gives consumers all the knobs they need, - which we try hard to keep stable. -shell/...: - The high-level API that gives consumers easy access to common - operations (e.g. create a file node from a reader without wrapping - with metadata). We work really hard to keep this stable. - -Then on top of the core/... and shell/... Go APIs, we have: - -cmd/...: - Command-line executables -test/...: - Integration tests, etc. - -To avoid cyclic imports, imports should never pull in higher-level -APIs into a lower-level package. For example, you could import all of -core and shell from cmd/... or test/..., but you couldn't import any -of shell from core/.... -*/ -package ipfs diff --git a/exchange/bitswap/bitswap.go b/exchange/bitswap/bitswap.go index 28b4b55..2b0227a 100644 --- a/exchange/bitswap/bitswap.go +++ b/exchange/bitswap/bitswap.go @@ -10,19 +10,18 @@ import ( "sync/atomic" "time" - blockstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - exchange "github.com/Casper-dev/Casper-server/exchange" - decision "github.com/Casper-dev/Casper-server/exchange/bitswap/decision" - bsmsg "github.com/Casper-dev/Casper-server/exchange/bitswap/message" - bsnet "github.com/Casper-dev/Casper-server/exchange/bitswap/network" - notifications "github.com/Casper-dev/Casper-server/exchange/bitswap/notifications" - flags "github.com/Casper-dev/Casper-server/flags" - "github.com/Casper-dev/Casper-server/thirdparty/delay" + blockstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + exchange "gitlab.com/casperDev/Casper-server/exchange" + decision "gitlab.com/casperDev/Casper-server/exchange/bitswap/decision" + bsmsg "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" + bsnet "gitlab.com/casperDev/Casper-server/exchange/bitswap/network" + notifications "gitlab.com/casperDev/Casper-server/exchange/bitswap/notifications" + flags "gitlab.com/casperDev/Casper-server/flags" + "gitlab.com/casperDev/Casper-server/thirdparty/delay" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" metrics "gx/ipfs/QmRg1gKTHzc3CZXSKzem8aR4E3TubFhbgXwfVuWnSK5CC5/go-metrics-interface" - process "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" procctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context" "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" @@ -211,6 +210,9 @@ func (bs *Bitswap) LedgerForPeer(p peer.ID) *decision.Receipt { // resources, provide a context with a reasonably short deadline (ie. not one // that lasts throughout the lifetime of the server) func (bs *Bitswap) GetBlocks(ctx context.Context, keys []*cid.Cid) (<-chan blocks.Block, error) { + for _, k := range keys { + log.Debugf("Want: %s", k.String()) + } if len(keys) == 0 { out := make(chan blocks.Block) close(out) diff --git a/exchange/bitswap/bitswap_test.go b/exchange/bitswap/bitswap_test.go index 5eca3ba..5845bfd 100644 --- a/exchange/bitswap/bitswap_test.go +++ b/exchange/bitswap/bitswap_test.go @@ -8,16 +8,17 @@ import ( "testing" "time" - blockstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - blocksutil "github.com/Casper-dev/Casper-server/blocks/blocksutil" - decision "github.com/Casper-dev/Casper-server/exchange/bitswap/decision" - tn "github.com/Casper-dev/Casper-server/exchange/bitswap/testnet" - mockrouting "github.com/Casper-dev/Casper-server/routing/mock" - delay "github.com/Casper-dev/Casper-server/thirdparty/delay" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" travis "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil/ci/travis" - detectrace "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-detect-race" + blockstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + blocksutil "gitlab.com/casperDev/Casper-server/blocks/blocksutil" + decision "gitlab.com/casperDev/Casper-server/exchange/bitswap/decision" + tn "gitlab.com/casperDev/Casper-server/exchange/bitswap/testnet" + mockrouting "gitlab.com/casperDev/Casper-server/routing/mock" + delay "gitlab.com/casperDev/Casper-server/thirdparty/delay" + + detectrace "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-detect-race" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" p2ptestutil "gx/ipfs/QmQGX417WoxKxDJeHqouMEmmH4G1RCENNSzkZYHrXy3Xb3/go-libp2p-netutil" diff --git a/exchange/bitswap/decision/bench_test.go b/exchange/bitswap/decision/bench_test.go index 8f53499..d810faa 100644 --- a/exchange/bitswap/decision/bench_test.go +++ b/exchange/bitswap/decision/bench_test.go @@ -5,11 +5,12 @@ import ( "math" "testing" - "github.com/Casper-dev/Casper-server/exchange/bitswap/wantlist" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" + + "gitlab.com/casperDev/Casper-server/exchange/bitswap/wantlist" ) // FWIW: At the time of this commit, including a timestamp in task increases diff --git a/exchange/bitswap/decision/engine.go b/exchange/bitswap/decision/engine.go index f8be655..4b71231 100644 --- a/exchange/bitswap/decision/engine.go +++ b/exchange/bitswap/decision/engine.go @@ -6,13 +6,13 @@ import ( "sync" "time" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - bsmsg "github.com/Casper-dev/Casper-server/exchange/bitswap/message" - wl "github.com/Casper-dev/Casper-server/exchange/bitswap/wantlist" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bsmsg "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" + wl "gitlab.com/casperDev/Casper-server/exchange/bitswap/wantlist" ) // TODO consider taking responsibility for other types of requests. For diff --git a/exchange/bitswap/decision/engine_test.go b/exchange/bitswap/decision/engine_test.go index 59da76b..301bfcd 100644 --- a/exchange/bitswap/decision/engine_test.go +++ b/exchange/bitswap/decision/engine_test.go @@ -9,13 +9,14 @@ import ( "testing" context "context" - blockstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - message "github.com/Casper-dev/Casper-server/exchange/bitswap/message" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" dssync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" + + blockstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + message "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" ) type peerAndEngine struct { diff --git a/exchange/bitswap/decision/ledger.go b/exchange/bitswap/decision/ledger.go index e0bed9c..bdfe1ba 100644 --- a/exchange/bitswap/decision/ledger.go +++ b/exchange/bitswap/decision/ledger.go @@ -4,13 +4,14 @@ import ( "sync" "time" - wl "github.com/Casper-dev/Casper-server/exchange/bitswap/wantlist" + wl "gitlab.com/casperDev/Casper-server/exchange/bitswap/wantlist" + + "gitlab.com/casperDev/Casper-SC/casper_sc" - "fmt" - "github.com/ethereum/go-ethereum/common" - "github.com/Casper-dev/Casper-SC/casper_sc" "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" + + "github.com/ethereum/go-ethereum/common" ) func newLedger(p peer.ID) *ledger { @@ -21,9 +22,33 @@ func newLedger(p peer.ID) *ledger { } } +var AllowedHashes map[string][]string + +func AllowHash(hash, wallet string) { + for _, w := range AllowedHashes[hash] { + if w == wallet { + return + } + } + AllowedHashes[hash] = append(AllowedHashes[hash], wallet) +} -var AllowedHashes map[string]bool -var Wallet string +func DisallowHash(hash, wallet string) { + for i, w := range AllowedHashes[hash] { + if w == wallet { + if len(AllowedHashes[hash]) == 1 { + delete(AllowedHashes, hash) + } else { + var tail []string + if i+1 < len(AllowedHashes[hash]) { + tail = AllowedHashes[hash][i+1:] + } + AllowedHashes[hash] = append(AllowedHashes[hash][:i], tail...) + } + return + } + } +} // ledger stores the data exchange relationship between two peers. // NOT threadsafe @@ -84,23 +109,20 @@ func (l *ledger) ReceivedBytes(n int) { } func (l *ledger) Wants(k *cid.Cid, priority int) { + l.wantList.Add(k, priority) + return log.Debugf("peer %s wants %s", l.Partner, k) - //fmt.Printf("peer %s wants %s\n", l.Partner.Pretty(), k.String()) - if AllowedHashes[k.String()] { - casperclient, _, _ := Casper_SC.GetSC() - isPre, _ := casperclient.IsPrepaid(nil, common.HexToAddress(Wallet)) - fmt.Println("is prepaid: ", isPre) + if wallets, ok := AllowedHashes[k.String()]; ok { + casperclient, _, _, _ := Casper_SC.GetSC() + + // TODO check add info to wantlist about wallet + isPre, _ := casperclient.IsPrepaid(nil, common.HexToAddress(wallets[0])) + log.Debugf("Wallet %s prepaid=%t", isPre) if isPre { l.wantList.Add(k, priority) - fmt.Printf("Hash %s was allowed to download w/ wallet %s\n", k.String(), Wallet) + log.Infof("Hash %s was allowed to download with wallet %s\n", k.String(), wallets[0]) } - } else { - - { - //fmt.Printf("Wanted %s hash, but not expected\n", k.String()) - } } - } func (l *ledger) CancelWant(k *cid.Cid) { diff --git a/exchange/bitswap/decision/peer_request_queue.go b/exchange/bitswap/decision/peer_request_queue.go index bc162cd..b75376a 100644 --- a/exchange/bitswap/decision/peer_request_queue.go +++ b/exchange/bitswap/decision/peer_request_queue.go @@ -4,8 +4,8 @@ import ( "sync" "time" - wantlist "github.com/Casper-dev/Casper-server/exchange/bitswap/wantlist" - pq "github.com/Casper-dev/Casper-server/thirdparty/pq" + wantlist "gitlab.com/casperDev/Casper-server/exchange/bitswap/wantlist" + pq "gitlab.com/casperDev/Casper-server/thirdparty/pq" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" diff --git a/exchange/bitswap/decision/peer_request_queue_test.go b/exchange/bitswap/decision/peer_request_queue_test.go index 56411aa..b9f9f19 100644 --- a/exchange/bitswap/decision/peer_request_queue_test.go +++ b/exchange/bitswap/decision/peer_request_queue_test.go @@ -8,10 +8,11 @@ import ( "strings" "testing" - "github.com/Casper-dev/Casper-server/exchange/bitswap/wantlist" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + + "gitlab.com/casperDev/Casper-server/exchange/bitswap/wantlist" ) func TestPushPop(t *testing.T) { diff --git a/exchange/bitswap/get.go b/exchange/bitswap/get.go index 5078203..32cc31d 100644 --- a/exchange/bitswap/get.go +++ b/exchange/bitswap/get.go @@ -4,10 +4,11 @@ import ( "context" "errors" - blockstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - notifications "github.com/Casper-dev/Casper-server/exchange/bitswap/notifications" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + blockstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + notifications "gitlab.com/casperDev/Casper-server/exchange/bitswap/notifications" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ) diff --git a/exchange/bitswap/message/message.go b/exchange/bitswap/message/message.go index 6aed75f..0878701 100644 --- a/exchange/bitswap/message/message.go +++ b/exchange/bitswap/message/message.go @@ -4,16 +4,22 @@ import ( "fmt" "io" - pb "github.com/Casper-dev/Casper-server/exchange/bitswap/message/pb" - wantlist "github.com/Casper-dev/Casper-server/exchange/bitswap/wantlist" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + bl "gitlab.com/casperDev/Casper-server/blocks" + uuid "gitlab.com/casperDev/Casper-server/casper/uuid" + pb "gitlab.com/casperDev/Casper-server/exchange/bitswap/message/pb" + wantlist "gitlab.com/casperDev/Casper-server/exchange/bitswap/wantlist" + inet "gx/ipfs/QmNa31VPzC561NWwRsJLE7nGYZYuuD2QfpK2b1q9BK54J1/go-libp2p-net" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" ggio "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/io" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" ) +var log = logging.Logger("bitswap.message") + // TODO move message.go into the bitswap package // TODO move bs/msg/internal/pb to bs/internal/pb and rename pb package to bitswap_pb @@ -84,7 +90,7 @@ func newMessageFromProto(pbm pb.Message) (BitSwapMessage, error) { // deprecated for _, d := range pbm.GetBlocks() { // CIDv0, sha256, protobuf only - b := blocks.NewBlock(d) + b := bl.NewBlockWithUUID(d) m.AddBlock(b) } // @@ -95,12 +101,17 @@ func newMessageFromProto(pbm pb.Message) (BitSwapMessage, error) { return nil, err } - c, err := pref.Sum(b.GetData()) + u, d := bl.SplitData(b.GetData()) + if uuid.IsUUIDNull(u) { + u = d + } + c, err := pref.Sum(u) + log.Debugf("CID: %s", c.String()) if err != nil { return nil, err } - blk, err := blocks.NewBlockWithCid(b.GetData(), c) + blk, err := bl.NewBlockWithCid(b.GetData(), c) if err != nil { return nil, err } diff --git a/exchange/bitswap/message/message_test.go b/exchange/bitswap/message/message_test.go index 6242b1b..09921a3 100644 --- a/exchange/bitswap/message/message_test.go +++ b/exchange/bitswap/message/message_test.go @@ -6,10 +6,11 @@ import ( proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" - pb "github.com/Casper-dev/Casper-server/exchange/bitswap/message/pb" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + + pb "gitlab.com/casperDev/Casper-server/exchange/bitswap/message/pb" ) func mkFakeCid(s string) *cid.Cid { diff --git a/exchange/bitswap/network/interface.go b/exchange/bitswap/network/interface.go index 25f87e9..07b096b 100644 --- a/exchange/bitswap/network/interface.go +++ b/exchange/bitswap/network/interface.go @@ -3,7 +3,7 @@ package network import ( "context" - bsmsg "github.com/Casper-dev/Casper-server/exchange/bitswap/message" + bsmsg "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" diff --git a/exchange/bitswap/network/ipfs_impl.go b/exchange/bitswap/network/ipfs_impl.go index 6e177f2..f45ae7e 100644 --- a/exchange/bitswap/network/ipfs_impl.go +++ b/exchange/bitswap/network/ipfs_impl.go @@ -6,7 +6,7 @@ import ( "io" "time" - bsmsg "github.com/Casper-dev/Casper-server/exchange/bitswap/message" + bsmsg "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" inet "gx/ipfs/QmNa31VPzC561NWwRsJLE7nGYZYuuD2QfpK2b1q9BK54J1/go-libp2p-net" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" diff --git a/exchange/bitswap/notifications/notifications.go b/exchange/bitswap/notifications/notifications.go index 243711f..1852848 100644 --- a/exchange/bitswap/notifications/notifications.go +++ b/exchange/bitswap/notifications/notifications.go @@ -5,8 +5,9 @@ import ( blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" - pubsub "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/briantigerchow/pubsub" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + + pubsub "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/briantigerchow/pubsub" ) const bufferSize = 16 diff --git a/exchange/bitswap/notifications/notifications_test.go b/exchange/bitswap/notifications/notifications_test.go index 6a50e42..f011afd 100644 --- a/exchange/bitswap/notifications/notifications_test.go +++ b/exchange/bitswap/notifications/notifications_test.go @@ -6,9 +6,10 @@ import ( "testing" "time" - blocksutil "github.com/Casper-dev/Casper-server/blocks/blocksutil" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + + blocksutil "gitlab.com/casperDev/Casper-server/blocks/blocksutil" ) func TestDuplicates(t *testing.T) { diff --git a/exchange/bitswap/session.go b/exchange/bitswap/session.go index 768270c..4e30004 100644 --- a/exchange/bitswap/session.go +++ b/exchange/bitswap/session.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - notifications "github.com/Casper-dev/Casper-server/exchange/bitswap/notifications" + notifications "gitlab.com/casperDev/Casper-server/exchange/bitswap/notifications" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" diff --git a/exchange/bitswap/session_test.go b/exchange/bitswap/session_test.go index 4b4b14f..0c7e2ce 100644 --- a/exchange/bitswap/session_test.go +++ b/exchange/bitswap/session_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - blocksutil "github.com/Casper-dev/Casper-server/blocks/blocksutil" + blocksutil "gitlab.com/casperDev/Casper-server/blocks/blocksutil" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" diff --git a/exchange/bitswap/testnet/interface.go b/exchange/bitswap/testnet/interface.go index 1353245..490e0a6 100644 --- a/exchange/bitswap/testnet/interface.go +++ b/exchange/bitswap/testnet/interface.go @@ -1,9 +1,10 @@ package bitswap import ( - bsnet "github.com/Casper-dev/Casper-server/exchange/bitswap/network" "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" + + bsnet "gitlab.com/casperDev/Casper-server/exchange/bitswap/network" ) type Network interface { diff --git a/exchange/bitswap/testnet/network_test.go b/exchange/bitswap/testnet/network_test.go index c084b4d..ae60165 100644 --- a/exchange/bitswap/testnet/network_test.go +++ b/exchange/bitswap/testnet/network_test.go @@ -5,13 +5,14 @@ import ( "testing" context "context" - bsmsg "github.com/Casper-dev/Casper-server/exchange/bitswap/message" - bsnet "github.com/Casper-dev/Casper-server/exchange/bitswap/network" - mockrouting "github.com/Casper-dev/Casper-server/routing/mock" - delay "github.com/Casper-dev/Casper-server/thirdparty/delay" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" + + bsmsg "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" + bsnet "gitlab.com/casperDev/Casper-server/exchange/bitswap/network" + mockrouting "gitlab.com/casperDev/Casper-server/routing/mock" + delay "gitlab.com/casperDev/Casper-server/thirdparty/delay" ) func TestSendMessageAsyncButWaitForResponse(t *testing.T) { diff --git a/exchange/bitswap/testnet/peernet.go b/exchange/bitswap/testnet/peernet.go index dcc56eb..5f90044 100644 --- a/exchange/bitswap/testnet/peernet.go +++ b/exchange/bitswap/testnet/peernet.go @@ -2,12 +2,13 @@ package bitswap import ( "context" - bsnet "github.com/Casper-dev/Casper-server/exchange/bitswap/network" - mockrouting "github.com/Casper-dev/Casper-server/routing/mock" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" mockpeernet "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/net/mock" + + bsnet "gitlab.com/casperDev/Casper-server/exchange/bitswap/network" + mockrouting "gitlab.com/casperDev/Casper-server/routing/mock" ) type peernet struct { diff --git a/exchange/bitswap/testnet/virtual.go b/exchange/bitswap/testnet/virtual.go index 5be362b..559d15d 100644 --- a/exchange/bitswap/testnet/virtual.go +++ b/exchange/bitswap/testnet/virtual.go @@ -4,10 +4,10 @@ import ( "context" "errors" - bsmsg "github.com/Casper-dev/Casper-server/exchange/bitswap/message" - bsnet "github.com/Casper-dev/Casper-server/exchange/bitswap/network" - mockrouting "github.com/Casper-dev/Casper-server/routing/mock" - delay "github.com/Casper-dev/Casper-server/thirdparty/delay" + bsmsg "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" + bsnet "gitlab.com/casperDev/Casper-server/exchange/bitswap/network" + mockrouting "gitlab.com/casperDev/Casper-server/routing/mock" + delay "gitlab.com/casperDev/Casper-server/thirdparty/delay" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" diff --git a/exchange/bitswap/testutils.go b/exchange/bitswap/testutils.go index c247f9b..2844bd4 100644 --- a/exchange/bitswap/testutils.go +++ b/exchange/bitswap/testutils.go @@ -4,12 +4,13 @@ import ( "context" "time" - blockstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - tn "github.com/Casper-dev/Casper-server/exchange/bitswap/testnet" - datastore2 "github.com/Casper-dev/Casper-server/thirdparty/datastore2" - delay "github.com/Casper-dev/Casper-server/thirdparty/delay" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + blockstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + tn "gitlab.com/casperDev/Casper-server/exchange/bitswap/testnet" + datastore2 "gitlab.com/casperDev/Casper-server/thirdparty/datastore2" + delay "gitlab.com/casperDev/Casper-server/thirdparty/delay" + p2ptestutil "gx/ipfs/QmQGX417WoxKxDJeHqouMEmmH4G1RCENNSzkZYHrXy3Xb3/go-libp2p-netutil" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" ds_sync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" diff --git a/exchange/bitswap/wantmanager.go b/exchange/bitswap/wantmanager.go index 101009c..561b6b5 100644 --- a/exchange/bitswap/wantmanager.go +++ b/exchange/bitswap/wantmanager.go @@ -5,10 +5,10 @@ import ( "sync" "time" - engine "github.com/Casper-dev/Casper-server/exchange/bitswap/decision" - bsmsg "github.com/Casper-dev/Casper-server/exchange/bitswap/message" - bsnet "github.com/Casper-dev/Casper-server/exchange/bitswap/network" - wantlist "github.com/Casper-dev/Casper-server/exchange/bitswap/wantlist" + engine "gitlab.com/casperDev/Casper-server/exchange/bitswap/decision" + bsmsg "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" + bsnet "gitlab.com/casperDev/Casper-server/exchange/bitswap/network" + wantlist "gitlab.com/casperDev/Casper-server/exchange/bitswap/wantlist" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" metrics "gx/ipfs/QmRg1gKTHzc3CZXSKzem8aR4E3TubFhbgXwfVuWnSK5CC5/go-metrics-interface" diff --git a/exchange/bitswap/workers.go b/exchange/bitswap/workers.go index fc655db..dee1380 100644 --- a/exchange/bitswap/workers.go +++ b/exchange/bitswap/workers.go @@ -6,7 +6,7 @@ import ( "sync" "time" - bsmsg "github.com/Casper-dev/Casper-server/exchange/bitswap/message" + bsmsg "gitlab.com/casperDev/Casper-server/exchange/bitswap/message" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" process "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" diff --git a/exchange/offline/offline.go b/exchange/offline/offline.go index a8c743e..8019eae 100644 --- a/exchange/offline/offline.go +++ b/exchange/offline/offline.go @@ -5,10 +5,11 @@ package offline import ( "context" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - exchange "github.com/Casper-dev/Casper-server/exchange" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + exchange "gitlab.com/casperDev/Casper-server/exchange" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ) diff --git a/exchange/offline/offline_test.go b/exchange/offline/offline_test.go index 26d4a48..63742a3 100644 --- a/exchange/offline/offline_test.go +++ b/exchange/offline/offline_test.go @@ -4,10 +4,11 @@ import ( "context" "testing" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - "github.com/Casper-dev/Casper-server/blocks/blocksutil" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + "gitlab.com/casperDev/Casper-server/blocks/blocksutil" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/exchange/reprovide/providers.go b/exchange/reprovide/providers.go index 062874f..c47679b 100644 --- a/exchange/reprovide/providers.go +++ b/exchange/reprovide/providers.go @@ -3,9 +3,9 @@ package reprovide import ( "context" - blocks "github.com/Casper-dev/Casper-server/blocks/blockstore" - merkledag "github.com/Casper-dev/Casper-server/merkledag" - pin "github.com/Casper-dev/Casper-server/pin" + blocks "gitlab.com/casperDev/Casper-server/blocks/blockstore" + merkledag "gitlab.com/casperDev/Casper-server/merkledag" + pin "gitlab.com/casperDev/Casper-server/pin" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ) diff --git a/exchange/reprovide/reprovide_test.go b/exchange/reprovide/reprovide_test.go index 5dbaa4c..869a113 100644 --- a/exchange/reprovide/reprovide_test.go +++ b/exchange/reprovide/reprovide_test.go @@ -4,15 +4,16 @@ import ( "testing" context "context" - blockstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - mock "github.com/Casper-dev/Casper-server/routing/mock" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" dssync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" - . "github.com/Casper-dev/Casper-server/exchange/reprovide" + blockstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + mock "gitlab.com/casperDev/Casper-server/routing/mock" + + . "gitlab.com/casperDev/Casper-server/exchange/reprovide" ) func TestReprovide(t *testing.T) { diff --git a/filestore/filestore.go b/filestore/filestore.go index 350131c..b960f27 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -10,10 +10,11 @@ package filestore import ( "context" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - posinfo "github.com/Casper-dev/Casper-server/thirdparty/posinfo" "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + posinfo "gitlab.com/casperDev/Casper-server/thirdparty/posinfo" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" dsq "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/query" diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index d5bcae6..994614a 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -7,9 +7,9 @@ import ( "math/rand" "testing" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - dag "github.com/Casper-dev/Casper-server/merkledag" - posinfo "github.com/Casper-dev/Casper-server/thirdparty/posinfo" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + dag "gitlab.com/casperDev/Casper-server/merkledag" + posinfo "gitlab.com/casperDev/Casper-server/thirdparty/posinfo" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index e1191e0..a5ab171 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -7,12 +7,13 @@ import ( "os" "path/filepath" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - pb "github.com/Casper-dev/Casper-server/filestore/pb" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" - posinfo "github.com/Casper-dev/Casper-server/thirdparty/posinfo" "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + pb "gitlab.com/casperDev/Casper-server/filestore/pb" + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" + posinfo "gitlab.com/casperDev/Casper-server/thirdparty/posinfo" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/filestore/util.go b/filestore/util.go index 9e88a1b..4654b6e 100644 --- a/filestore/util.go +++ b/filestore/util.go @@ -4,9 +4,9 @@ import ( "fmt" "sort" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - pb "github.com/Casper-dev/Casper-server/filestore/pb" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + pb "gitlab.com/casperDev/Casper-server/filestore/pb" + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/fuse/ipns/common.go b/fuse/ipns/common.go index 2d3d14e..7d83434 100644 --- a/fuse/ipns/common.go +++ b/fuse/ipns/common.go @@ -3,11 +3,12 @@ package ipns import ( context "context" - "github.com/Casper-dev/Casper-server/core" - nsys "github.com/Casper-dev/Casper-server/namesys" - path "github.com/Casper-dev/Casper-server/path" - ft "github.com/Casper-dev/Casper-server/unixfs" ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" + + "gitlab.com/casperDev/Casper-server/core" + nsys "gitlab.com/casperDev/Casper-server/namesys" + path "gitlab.com/casperDev/Casper-server/path" + ft "gitlab.com/casperDev/Casper-server/unixfs" ) // InitializeKeyspace sets the ipns record for the given key to diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index 13f22f5..4019cd8 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -12,12 +12,13 @@ import ( "sync" "testing" - core "github.com/Casper-dev/Casper-server/core" - namesys "github.com/Casper-dev/Casper-server/namesys" - offroute "github.com/Casper-dev/Casper-server/routing/offline" ci "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil/ci" - racedet "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-detect-race" + core "gitlab.com/casperDev/Casper-server/core" + namesys "gitlab.com/casperDev/Casper-server/namesys" + offroute "gitlab.com/casperDev/Casper-server/routing/offline" + + racedet "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-detect-race" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" fstest "gx/ipfs/QmaFNtBAXX4nVMQWbUqNysXyhevUj1k4B1y5uS45LC7Vw9/fuse/fs/fstestutil" diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 0fa76e0..7fd7008 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -11,12 +11,12 @@ import ( "io" "os" - core "github.com/Casper-dev/Casper-server/core" - dag "github.com/Casper-dev/Casper-server/merkledag" - mfs "github.com/Casper-dev/Casper-server/mfs" - namesys "github.com/Casper-dev/Casper-server/namesys" - path "github.com/Casper-dev/Casper-server/path" - ft "github.com/Casper-dev/Casper-server/unixfs" + core "gitlab.com/casperDev/Casper-server/core" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mfs "gitlab.com/casperDev/Casper-server/mfs" + namesys "gitlab.com/casperDev/Casper-server/namesys" + path "gitlab.com/casperDev/Casper-server/path" + ft "gitlab.com/casperDev/Casper-server/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" diff --git a/fuse/ipns/mount_unix.go b/fuse/ipns/mount_unix.go index 772a11e..bcf5bb1 100644 --- a/fuse/ipns/mount_unix.go +++ b/fuse/ipns/mount_unix.go @@ -4,8 +4,8 @@ package ipns import ( - core "github.com/Casper-dev/Casper-server/core" - mount "github.com/Casper-dev/Casper-server/fuse/mount" + core "gitlab.com/casperDev/Casper-server/core" + mount "gitlab.com/casperDev/Casper-server/fuse/mount" ) // Mount mounts ipns at a given location, and returns a mount.Mount instance. diff --git a/fuse/node/mount_darwin.go b/fuse/node/mount_darwin.go index dd856cc..b4551c0 100644 --- a/fuse/node/mount_darwin.go +++ b/fuse/node/mount_darwin.go @@ -10,7 +10,7 @@ import ( "strings" "syscall" - core "github.com/Casper-dev/Casper-server/core" + core "gitlab.com/casperDev/Casper-server/core" "gx/ipfs/QmYRGECuvQnRX73fcvPnGbYijBcGN2HbKZQ7jh26qmLiHG/semver" ) diff --git a/fuse/node/mount_nofuse.go b/fuse/node/mount_nofuse.go index 810d86c..c33eb0d 100644 --- a/fuse/node/mount_nofuse.go +++ b/fuse/node/mount_nofuse.go @@ -6,7 +6,7 @@ package node import ( "errors" - core "github.com/Casper-dev/Casper-server/core" + core "gitlab.com/casperDev/Casper-server/core" ) func Mount(node *core.IpfsNode, fsdir, nsdir string) error { diff --git a/fuse/node/mount_test.go b/fuse/node/mount_test.go index a631d5a..375837f 100644 --- a/fuse/node/mount_test.go +++ b/fuse/node/mount_test.go @@ -10,12 +10,13 @@ import ( "time" context "context" - core "github.com/Casper-dev/Casper-server/core" - ipns "github.com/Casper-dev/Casper-server/fuse/ipns" - mount "github.com/Casper-dev/Casper-server/fuse/mount" - namesys "github.com/Casper-dev/Casper-server/namesys" - offroute "github.com/Casper-dev/Casper-server/routing/offline" ci "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil/ci" + + core "gitlab.com/casperDev/Casper-server/core" + ipns "gitlab.com/casperDev/Casper-server/fuse/ipns" + mount "gitlab.com/casperDev/Casper-server/fuse/mount" + namesys "gitlab.com/casperDev/Casper-server/namesys" + offroute "gitlab.com/casperDev/Casper-server/routing/offline" ) func maybeSkipFuseTests(t *testing.T) { diff --git a/fuse/node/mount_unix.go b/fuse/node/mount_unix.go index 20d1698..c965d6d 100644 --- a/fuse/node/mount_unix.go +++ b/fuse/node/mount_unix.go @@ -9,11 +9,12 @@ import ( "strings" "sync" - core "github.com/Casper-dev/Casper-server/core" - ipns "github.com/Casper-dev/Casper-server/fuse/ipns" - mount "github.com/Casper-dev/Casper-server/fuse/mount" - rofs "github.com/Casper-dev/Casper-server/fuse/readonly" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + + core "gitlab.com/casperDev/Casper-server/core" + ipns "gitlab.com/casperDev/Casper-server/fuse/ipns" + mount "gitlab.com/casperDev/Casper-server/fuse/mount" + rofs "gitlab.com/casperDev/Casper-server/fuse/readonly" ) var log = logging.Logger("node") diff --git a/fuse/node/mount_windows.go b/fuse/node/mount_windows.go index 898e0fa..fc21864 100644 --- a/fuse/node/mount_windows.go +++ b/fuse/node/mount_windows.go @@ -1,7 +1,7 @@ package node import ( - "github.com/Casper-dev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core" ) func Mount(node *core.IpfsNode, fsdir, nsdir string) error { diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index 6407a22..daa89f9 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -13,15 +13,16 @@ import ( "sync" "testing" - core "github.com/Casper-dev/Casper-server/core" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" - coremock "github.com/Casper-dev/Casper-server/core/mock" - importer "github.com/Casper-dev/Casper-server/importer" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - dag "github.com/Casper-dev/Casper-server/merkledag" - uio "github.com/Casper-dev/Casper-server/unixfs/io" ci "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil/ci" + core "gitlab.com/casperDev/Casper-server/core" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + coremock "gitlab.com/casperDev/Casper-server/core/mock" + importer "gitlab.com/casperDev/Casper-server/importer" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + dag "gitlab.com/casperDev/Casper-server/merkledag" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" fstest "gx/ipfs/QmaFNtBAXX4nVMQWbUqNysXyhevUj1k4B1y5uS45LC7Vw9/fuse/fs/fstestutil" diff --git a/fuse/readonly/mount_unix.go b/fuse/readonly/mount_unix.go index 7a62a97..95bffd3 100644 --- a/fuse/readonly/mount_unix.go +++ b/fuse/readonly/mount_unix.go @@ -4,8 +4,8 @@ package readonly import ( - core "github.com/Casper-dev/Casper-server/core" - mount "github.com/Casper-dev/Casper-server/fuse/mount" + core "gitlab.com/casperDev/Casper-server/core" + mount "gitlab.com/casperDev/Casper-server/fuse/mount" ) // Mount mounts IPFS at a given location, and returns a mount.Mount instance. diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 701bd2c..79cb83d 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -10,11 +10,11 @@ import ( "os" "syscall" - core "github.com/Casper-dev/Casper-server/core" - mdag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" - uio "github.com/Casper-dev/Casper-server/unixfs/io" - ftpb "github.com/Casper-dev/Casper-server/unixfs/pb" + core "gitlab.com/casperDev/Casper-server/core" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + ftpb "gitlab.com/casperDev/Casper-server/unixfs/pb" format "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" diff --git a/importer/balanced/balanced_test.go b/importer/balanced/balanced_test.go index 21ac936..234e24a 100644 --- a/importer/balanced/balanced_test.go +++ b/importer/balanced/balanced_test.go @@ -8,11 +8,11 @@ import ( mrand "math/rand" "testing" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - h "github.com/Casper-dev/Casper-server/importer/helpers" - dag "github.com/Casper-dev/Casper-server/merkledag" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + h "gitlab.com/casperDev/Casper-server/importer/helpers" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" "context" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/importer/balanced/builder.go b/importer/balanced/builder.go index 72a15c0..5e7eebf 100644 --- a/importer/balanced/builder.go +++ b/importer/balanced/builder.go @@ -3,12 +3,15 @@ package balanced import ( "errors" - h "github.com/Casper-dev/Casper-server/importer/helpers" - node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + + h "gitlab.com/casperDev/Casper-server/importer/helpers" ) -func BalancedLayout(db *h.DagBuilderHelper) (node.Node, error) { +var log = logging.Logger("balanced.builder") + +func BalancedLayout(db *h.DagBuilderHelper, uuidList ...[]byte) (node.Node, error) { var offset uint64 = 0 var root *h.UnixfsNode for level := 0; !db.Done(); level++ { @@ -30,12 +33,15 @@ func BalancedLayout(db *h.DagBuilderHelper) (node.Node, error) { offset = nroot.FileSize() root = nroot - } if root == nil { root = db.NewUnixfsNode() } + if len(uuidList) > 0 { + root.SetUUID(uuidList[0]) + } + out, err := db.Add(root) if err != nil { return nil, err diff --git a/importer/chunk/rabin.go b/importer/chunk/rabin.go index 0b94c36..3371da3 100644 --- a/importer/chunk/rabin.go +++ b/importer/chunk/rabin.go @@ -4,7 +4,7 @@ import ( "hash/fnv" "io" - "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/whyrusleeping/chunker" + "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/whyrusleeping/chunker" ) var IpfsRabinPoly = chunker.Pol(17437180132763653) diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 7db45b1..9edd047 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -4,10 +4,11 @@ import ( "io" "os" - "github.com/Casper-dev/Casper-server/commands/files" - "github.com/Casper-dev/Casper-server/importer/chunk" - dag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" + uuid "gitlab.com/casperDev/Casper-server/casper/uuid" + "gitlab.com/casperDev/Casper-server/commands/files" + "gitlab.com/casperDev/Casper-server/importer/chunk" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" @@ -112,8 +113,12 @@ func (db *DagBuilderHelper) GetDagServ() dag.DAGService { // NewUnixfsNode creates a new Unixfs node to represent a file. func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode { + node := new(dag.ProtoNode) + ///node.SetUUID(uuid.GenUUID()) + node.SetUUID(uuid.NullUUID) + n := &UnixfsNode{ - node: new(dag.ProtoNode), + node: node, ufmt: &ft.FSNode{Type: ft.TFile}, } n.SetPrefix(db.prefix) @@ -122,8 +127,12 @@ func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode { // newUnixfsBlock creates a new Unixfs node to represent a raw data block func (db *DagBuilderHelper) newUnixfsBlock() *UnixfsNode { + node := new(dag.ProtoNode) + ///node.SetUUID(uuid.GenUUID()) + node.SetUUID(uuid.NullUUID) + n := &UnixfsNode{ - node: new(dag.ProtoNode), + node: node, ufmt: &ft.FSNode{Type: ft.TRaw}, } n.SetPrefix(db.prefix) @@ -194,6 +203,7 @@ func (db *DagBuilderHelper) SetPosInfo(node *UnixfsNode, offset uint64) { } func (db *DagBuilderHelper) Add(node *UnixfsNode) (node.Node, error) { + log.Debugf("db add node %+v", node) dn, err := node.GetDagNode() if err != nil { return nil, err diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 8c88662..b1c1747 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -5,14 +5,19 @@ import ( "fmt" "os" - dag "github.com/Casper-dev/Casper-server/merkledag" - pi "github.com/Casper-dev/Casper-server/thirdparty/posinfo" - ft "github.com/Casper-dev/Casper-server/unixfs" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + + uuid "gitlab.com/casperDev/Casper-server/casper/uuid" + dag "gitlab.com/casperDev/Casper-server/merkledag" + pi "gitlab.com/casperDev/Casper-server/thirdparty/posinfo" + ft "gitlab.com/casperDev/Casper-server/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) +var log = logging.Logger("imp.helpers") + // BlockSizeLimit specifies the maximum size an imported block can have. var BlockSizeLimit = 1048576 // 1 MB @@ -65,6 +70,10 @@ func (n *UnixfsNode) SetPrefix(prefix *cid.Prefix) { n.node.SetPrefix(prefix) } +func (n *UnixfsNode) SetUUID(uuid []byte) { + n.node.SetUUID(uuid) +} + func (n *UnixfsNode) NumChildren() int { return n.ufmt.NumChildren() } @@ -127,7 +136,7 @@ func (n *UnixfsNode) SetData(data []byte) { func (n *UnixfsNode) FileSize() uint64 { if n.raw { - return uint64(len(n.rawnode.RawData())) + return uint64(len(n.rawnode.RawData()) - uuid.UUIDLen) } return n.ufmt.FileSize() } diff --git a/importer/importer.go b/importer/importer.go index 66133ef..db3dca5 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -6,12 +6,12 @@ import ( "fmt" "os" - "github.com/Casper-dev/Casper-server/commands/files" - bal "github.com/Casper-dev/Casper-server/importer/balanced" - "github.com/Casper-dev/Casper-server/importer/chunk" - h "github.com/Casper-dev/Casper-server/importer/helpers" - trickle "github.com/Casper-dev/Casper-server/importer/trickle" - dag "github.com/Casper-dev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/commands/files" + bal "gitlab.com/casperDev/Casper-server/importer/balanced" + "gitlab.com/casperDev/Casper-server/importer/chunk" + h "gitlab.com/casperDev/Casper-server/importer/helpers" + trickle "gitlab.com/casperDev/Casper-server/importer/trickle" + dag "gitlab.com/casperDev/Casper-server/merkledag" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) @@ -28,7 +28,7 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (node.Node, error) { return nil, fmt.Errorf("`%s` is a directory", fpath) } - f, err := files.NewSerialFile(fpath, fpath, false, stat) + f, err := files.NewSerialFile(fpath, fpath, false, stat, nil) if err != nil { return nil, err } diff --git a/importer/importer_test.go b/importer/importer_test.go index c6c327a..0f52100 100644 --- a/importer/importer_test.go +++ b/importer/importer_test.go @@ -7,10 +7,10 @@ import ( "io/ioutil" "testing" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - dag "github.com/Casper-dev/Casper-server/merkledag" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/importer/trickle/trickle_test.go b/importer/trickle/trickle_test.go index 9ed3031..1da8966 100644 --- a/importer/trickle/trickle_test.go +++ b/importer/trickle/trickle_test.go @@ -9,12 +9,12 @@ import ( mrand "math/rand" "testing" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - h "github.com/Casper-dev/Casper-server/importer/helpers" - merkledag "github.com/Casper-dev/Casper-server/merkledag" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" - ft "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + h "gitlab.com/casperDev/Casper-server/importer/helpers" + merkledag "gitlab.com/casperDev/Casper-server/merkledag" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" ) diff --git a/importer/trickle/trickledag.go b/importer/trickle/trickledag.go index e4d7ca8..f66bb74 100644 --- a/importer/trickle/trickledag.go +++ b/importer/trickle/trickledag.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - h "github.com/Casper-dev/Casper-server/importer/helpers" - dag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" + h "gitlab.com/casperDev/Casper-server/importer/helpers" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/merkledag/coding.go b/merkledag/coding.go index 5233a1c..d1bf10d 100644 --- a/merkledag/coding.go +++ b/merkledag/coding.go @@ -7,10 +7,13 @@ import ( "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" - pb "github.com/Casper-dev/Casper-server/merkledag/pb" + pb "gitlab.com/casperDev/Casper-server/merkledag/pb" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" + + bl "gitlab.com/casperDev/Casper-server/blocks" + uid "gitlab.com/casperDev/Casper-server/casper/uuid" ) // for now, we use a PBNode intermediate thing. @@ -78,6 +81,7 @@ func (n *ProtoNode) getPBNode() *pb.PBNode { // It may use a cached encoded version, unless the force flag is given. func (n *ProtoNode) EncodeProtobuf(force bool) ([]byte, error) { sort.Stable(LinkSlice(n.links)) // keep links sorted + if n.encoded == nil || force { n.cached = nil var err error @@ -91,7 +95,12 @@ func (n *ProtoNode) EncodeProtobuf(force bool) ([]byte, error) { if n.Prefix.Codec == 0 { // unset n.Prefix = v0CidPrefix } - c, err := n.Prefix.Sum(n.encoded) + + id := n.uuid + if uid.IsUUIDNull(n.uuid) { + id = n.encoded + } + c, err := n.Prefix.Sum(id) if err != nil { return nil, err } @@ -116,11 +125,14 @@ func DecodeProtobuf(encoded []byte) (*ProtoNode, error) { // node.DecodeBlockFunc func DecodeProtobufBlock(b blocks.Block) (node.Node, error) { c := b.Cid() + if c.Type() != cid.DagProtobuf { return nil, fmt.Errorf("this function can only decode protobuf nodes") } - decnd, err := DecodeProtobuf(b.RawData()) + uuid, data := bl.SplitData(b.RawData()) + decnd, err := DecodeProtobuf(data) + if err != nil { if strings.Contains(err.Error(), "Unmarshal failed") { return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", c) @@ -128,6 +140,7 @@ func DecodeProtobufBlock(b blocks.Block) (node.Node, error) { return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err) } + decnd.SetUUID(uuid) decnd.cached = c decnd.Prefix = c.Prefix() return decnd, nil diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index a5203a8..1bcd32b 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -4,14 +4,16 @@ package merkledag import ( "context" "fmt" + "runtime/debug" "sync" - bserv "github.com/Casper-dev/Casper-server/blockservice" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" + ipldcbor "gx/ipfs/QmWCs8kMecJwCPK8JThue8TjgM2ieJ2HjTLDu7Cv2NEmZi/go-ipld-cbor" + + bserv "gitlab.com/casperDev/Casper-server/blockservice" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" ) // TODO: We should move these registrations elsewhere. Really, most of the IPLD @@ -95,6 +97,11 @@ func (n *dagService) Get(ctx context.Context, c *cid.Cid) (node.Node, error) { defer cancel() b, err := n.Blocks.GetBlock(ctx, c) + if b == nil { + debug.PrintStack() + err = ErrNotFound + } + if err != nil { if err == bserv.ErrNotFound { return nil, ErrNotFound diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 869620c..f996f19 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -13,17 +13,18 @@ import ( "testing" "time" - bserv "github.com/Casper-dev/Casper-server/blockservice" - bstest "github.com/Casper-dev/Casper-server/blockservice/test" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - imp "github.com/Casper-dev/Casper-server/importer" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - . "github.com/Casper-dev/Casper-server/merkledag" - mdpb "github.com/Casper-dev/Casper-server/merkledag/pb" - dstest "github.com/Casper-dev/Casper-server/merkledag/test" - uio "github.com/Casper-dev/Casper-server/unixfs/io" blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + bserv "gitlab.com/casperDev/Casper-server/blockservice" + bstest "gitlab.com/casperDev/Casper-server/blockservice/test" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + imp "gitlab.com/casperDev/Casper-server/importer" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + . "gitlab.com/casperDev/Casper-server/merkledag" + mdpb "gitlab.com/casperDev/Casper-server/merkledag/pb" + dstest "gitlab.com/casperDev/Casper-server/merkledag/test" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/merkledag/node.go b/merkledag/node.go index ad021fa..9114f52 100644 --- a/merkledag/node.go +++ b/merkledag/node.go @@ -5,19 +5,28 @@ import ( "encoding/json" "fmt" + bl "gitlab.com/casperDev/Casper-server/blocks" + uid "gitlab.com/casperDev/Casper-server/casper/uuid" + + "bytes" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58" mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" ) var ErrNotProtobuf = fmt.Errorf("expected protobuf dag node") var ErrLinkNotFound = fmt.Errorf("no link by that name") +var log = logging.Logger("dag.node") + // Node represents a node in the IPFS Merkle DAG. // nodes have opaque data and a set of navigable links. type ProtoNode struct { links []*node.Link data []byte + uuid []byte // cache encoded/marshaled value encoded []byte @@ -79,8 +88,8 @@ func (ls LinkSlice) Len() int { return len(ls) } func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] } func (ls LinkSlice) Less(a, b int) bool { return ls[a].Name < ls[b].Name } -func NodeWithData(d []byte) *ProtoNode { - return &ProtoNode{data: d} +func NodeWithData(data []byte) *ProtoNode { + return &ProtoNode{data: data} } // AddNodeLink adds a link to another node. @@ -199,24 +208,46 @@ func (n *ProtoNode) Copy() node.Node { nnode.Prefix = n.Prefix + if len(n.uuid) > 0 { + nnode.uuid = make([]byte, len(n.uuid)) + copy(nnode.uuid, n.uuid) + } + return nnode } func (n *ProtoNode) RawData() []byte { out, _ := n.EncodeProtobuf(false) - return out + if uid.IsUUIDNull(n.uuid) { + return append(uid.NullUUID, out...) + } + + return append(n.uuid, out...) } func (n *ProtoNode) Data() []byte { return n.data } +func (n *ProtoNode) UUID() []byte { + if n.uuid == nil { + return uid.NullUUID + } + + return n.uuid +} + func (n *ProtoNode) SetData(d []byte) { n.encoded = nil n.cached = nil n.data = d } +func (n *ProtoNode) SetUUID(uuid []byte) { + n.cached = nil + n.uuid = uuid +} + // UpdateNodeLink return a copy of the node with the link name set to point to // that. If a link of the same name existed, it is removed. func (n *ProtoNode) UpdateNodeLink(name string, that *ProtoNode) (*ProtoNode, error) { @@ -257,7 +288,7 @@ func (n *ProtoNode) Stat() (*node.NodeStat, error) { Hash: n.Cid().String(), NumLinks: len(n.links), BlockSize: len(enc), - LinksSize: len(enc) - len(n.data), // includes framing. + LinksSize: len(enc) - len(n.data), // - uid.UUIDLen, // includes framing. DataSize: len(n.data), CumulativeSize: int(cumSize), }, nil @@ -273,22 +304,32 @@ func (n *ProtoNode) UnmarshalJSON(b []byte) error { s := struct { Data []byte `json:"data"` Links []*node.Link `json:"links"` + UUID string `json:"uuid,omitempty"` }{} + log.Debugf("JSON %s", b) err := json.Unmarshal(b, &s) if err != nil { return err } - n.data = s.Data n.links = s.Links + n.data = s.Data + n.uuid = base58.Decode(s.UUID) + return nil } func (n *ProtoNode) MarshalJSON() ([]byte, error) { + uuid := n.uuid + if uuid == nil { + uuid = uid.NullUUID + } + out := map[string]interface{}{ "data": n.data, "links": n.links, + "uuid": base58.Encode(uuid), } return json.Marshal(out) @@ -303,7 +344,14 @@ func (n *ProtoNode) Cid() *cid.Cid { n.SetPrefix(nil) } - c, err := n.Prefix.Sum(n.RawData()) + id := n.uuid + if id == nil || bytes.Equal(id, uid.NullUUID) { + _, id = bl.SplitData(n.RawData()) + } + + //c, err := n.Prefix.Sum(n.RawData()) + c, err := n.Prefix.Sum(id) + if err != nil { // programmer error err = fmt.Errorf("invalid CID of length %d: %x: %v", len(n.RawData()), n.RawData(), err) diff --git a/merkledag/node_test.go b/merkledag/node_test.go index 621cb17..d71fe38 100644 --- a/merkledag/node_test.go +++ b/merkledag/node_test.go @@ -5,8 +5,8 @@ import ( "context" "testing" - . "github.com/Casper-dev/Casper-server/merkledag" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" + . "gitlab.com/casperDev/Casper-server/merkledag" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/merkledag/pb/merkledag.pb.go b/merkledag/pb/merkledag.pb.go index a4c7358..60ad041 100644 --- a/merkledag/pb/merkledag.pb.go +++ b/merkledag/pb/merkledag.pb.go @@ -264,6 +264,7 @@ func (m *PBNode) Unmarshal(data []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } + m.Data = append([]byte{}, data[index:postIndex]...) index = postIndex default: diff --git a/merkledag/raw.go b/merkledag/raw.go index e73fb88..0de62f7 100644 --- a/merkledag/raw.go +++ b/merkledag/raw.go @@ -2,11 +2,16 @@ package merkledag import ( "fmt" + + bl "gitlab.com/casperDev/Casper-server/blocks" + "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" + + uuid "gitlab.com/casperDev/Casper-server/casper/uuid" ) type RawNode struct { @@ -16,9 +21,13 @@ type RawNode struct { // NewRawNode creates a RawNode using the default sha2-256 hash // funcition. func NewRawNode(data []byte) *RawNode { + //id := uuid.GenUUID() + //h := u.Hash(id) h := u.Hash(data) c := cid.NewCidV1(cid.Raw, h) - blk, _ := blocks.NewBlockWithCid(data, c) + + log.Debugf("NewRawNode with CID %s", c.String()) + blk, _ := bl.NewBlockWithCid(append(uuid.NullUUID, data...), c) return &RawNode{blk} } @@ -41,11 +50,17 @@ func NewRawNodeWPrefix(data []byte, prefix cid.Prefix) (*RawNode, error) { if prefix.Version == 0 { prefix.Version = 1 } + + ///id := uuid.GenUUID() + ///c, err := prefix.Sum(id) + id := uuid.NullUUID c, err := prefix.Sum(data) + log.Debugf("NewRawNodeWPrefix with CID %s", c.String()) if err != nil { return nil, err } - blk, err := blocks.NewBlockWithCid(data, c) + + blk, err := blocks.NewBlockWithCid(append(id, data...), c) if err != nil { return nil, err } @@ -87,7 +102,7 @@ func (rn *RawNode) Size() (uint64, error) { func (rn *RawNode) Stat() (*node.NodeStat, error) { return &node.NodeStat{ CumulativeSize: len(rn.RawData()), - DataSize: len(rn.RawData()), + DataSize: len(rn.RawData()) - uuid.UUIDLen, }, nil } diff --git a/merkledag/test/utils.go b/merkledag/test/utils.go index 13fd5f8..dc598c4 100644 --- a/merkledag/test/utils.go +++ b/merkledag/test/utils.go @@ -1,12 +1,13 @@ package mdutils import ( - "github.com/Casper-dev/Casper-server/blocks/blockstore" - bsrv "github.com/Casper-dev/Casper-server/blockservice" - "github.com/Casper-dev/Casper-server/exchange/offline" - dag "github.com/Casper-dev/Casper-server/merkledag" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" dssync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" + + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bsrv "gitlab.com/casperDev/Casper-server/blockservice" + "gitlab.com/casperDev/Casper-server/exchange/offline" + dag "gitlab.com/casperDev/Casper-server/merkledag" ) func Mock() dag.DAGService { diff --git a/merkledag/traverse/traverse_test.go b/merkledag/traverse/traverse_test.go index cde15c3..ff9fff8 100644 --- a/merkledag/traverse/traverse_test.go +++ b/merkledag/traverse/traverse_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" - mdag "github.com/Casper-dev/Casper-server/merkledag" - mdagtest "github.com/Casper-dev/Casper-server/merkledag/test" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + mdagtest "gitlab.com/casperDev/Casper-server/merkledag/test" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/merkledag/utils/diff.go b/merkledag/utils/diff.go index cc96c8d..1f63702 100644 --- a/merkledag/utils/diff.go +++ b/merkledag/utils/diff.go @@ -5,7 +5,7 @@ import ( "fmt" "path" - dag "github.com/Casper-dev/Casper-server/merkledag" + dag "gitlab.com/casperDev/Casper-server/merkledag" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/merkledag/utils/diffenum.go b/merkledag/utils/diffenum.go index 9ff9724..d7dd156 100644 --- a/merkledag/utils/diffenum.go +++ b/merkledag/utils/diffenum.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - mdag "github.com/Casper-dev/Casper-server/merkledag" + mdag "gitlab.com/casperDev/Casper-server/merkledag" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/merkledag/utils/diffenum_test.go b/merkledag/utils/diffenum_test.go index 3c761ab..9f88177 100644 --- a/merkledag/utils/diffenum_test.go +++ b/merkledag/utils/diffenum_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" - dag "github.com/Casper-dev/Casper-server/merkledag" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/merkledag/utils/utils.go b/merkledag/utils/utils.go index 40a7f90..df0954a 100644 --- a/merkledag/utils/utils.go +++ b/merkledag/utils/utils.go @@ -4,11 +4,11 @@ import ( "context" "errors" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - bserv "github.com/Casper-dev/Casper-server/blockservice" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - dag "github.com/Casper-dev/Casper-server/merkledag" - path "github.com/Casper-dev/Casper-server/path" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bserv "gitlab.com/casperDev/Casper-server/blockservice" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + dag "gitlab.com/casperDev/Casper-server/merkledag" + path "gitlab.com/casperDev/Casper-server/path" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/merkledag/utils/utils_test.go b/merkledag/utils/utils_test.go index 76654af..d153711 100644 --- a/merkledag/utils/utils_test.go +++ b/merkledag/utils/utils_test.go @@ -3,9 +3,9 @@ package dagutils import ( "testing" - dag "github.com/Casper-dev/Casper-server/merkledag" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" - path "github.com/Casper-dev/Casper-server/path" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" + path "gitlab.com/casperDev/Casper-server/path" context "context" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" diff --git a/mfs/dir.go b/mfs/dir.go index f677c3f..8c6ddc2 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -10,10 +10,10 @@ import ( "sync" "time" - dag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" - ufspb "github.com/Casper-dev/Casper-server/unixfs/pb" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + ufspb "gitlab.com/casperDev/Casper-server/unixfs/pb" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/mfs/fd.go b/mfs/fd.go index 6b13123..65b6fa6 100644 --- a/mfs/fd.go +++ b/mfs/fd.go @@ -4,7 +4,7 @@ import ( "fmt" "io" - mod "github.com/Casper-dev/Casper-server/unixfs/mod" + mod "gitlab.com/casperDev/Casper-server/unixfs/mod" context "context" ) diff --git a/mfs/file.go b/mfs/file.go index f36351d..6bef8e2 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -5,10 +5,10 @@ import ( "fmt" "sync" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - dag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" - mod "github.com/Casper-dev/Casper-server/unixfs/mod" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + mod "gitlab.com/casperDev/Casper-server/unixfs/mod" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 8608fa2..e840a2b 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -14,15 +14,15 @@ import ( "testing" "time" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - bserv "github.com/Casper-dev/Casper-server/blockservice" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - importer "github.com/Casper-dev/Casper-server/importer" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - dag "github.com/Casper-dev/Casper-server/merkledag" - "github.com/Casper-dev/Casper-server/path" - ft "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bserv "gitlab.com/casperDev/Casper-server/blockservice" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + importer "gitlab.com/casperDev/Casper-server/importer" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + dag "gitlab.com/casperDev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/path" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/mfs/ops.go b/mfs/ops.go index 4cdcbfa..4353ed0 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -7,7 +7,7 @@ import ( gopath "path" "strings" - path "github.com/Casper-dev/Casper-server/path" + path "gitlab.com/casperDev/Casper-server/path" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/mfs/system.go b/mfs/system.go index 9197642..25cb063 100644 --- a/mfs/system.go +++ b/mfs/system.go @@ -16,8 +16,8 @@ import ( "sync" "time" - dag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/mk/golang.mk b/mk/golang.mk index 1abf4ea..c0a9bad 100644 --- a/mk/golang.mk +++ b/mk/golang.mk @@ -11,7 +11,7 @@ DEPS_GO := TEST_GO := CHECK_GO := -go-pkg-name=$(shell $(GOCC) list $(go-tags) github.com/ipfs/go-ipfs/$(1)) +go-pkg-name=$(shell $(GOCC) list $(go-tags) gitlab.com/casperDev/Casper-server/$(1)) go-main-name=$(notdir $(call go-pkg-name,$(1)))$(?exe) go-curr-pkg-tgt=$(d)/$(call go-main-name,$(d)) go-pkgs-novendor=$(shell $(GOCC) list github.com/ipfs/go-ipfs/... | grep -v /Godeps/) diff --git a/namesys/base.go b/namesys/base.go index 457d76d..879dcfa 100644 --- a/namesys/base.go +++ b/namesys/base.go @@ -5,7 +5,7 @@ import ( context "context" - path "github.com/Casper-dev/Casper-server/path" + path "gitlab.com/casperDev/Casper-server/path" ) type resolver interface { diff --git a/namesys/dns.go b/namesys/dns.go index abeb50a..829648e 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -6,7 +6,7 @@ import ( "net" "strings" - path "github.com/Casper-dev/Casper-server/path" + path "gitlab.com/casperDev/Casper-server/path" isd "gx/ipfs/QmZmmuAXgX73UQmX1jRKjTGmjzq24Jinqkq8vzkBtno4uX/go-is-domain" ) diff --git a/namesys/interface.go b/namesys/interface.go index 87a9daa..773ff28 100644 --- a/namesys/interface.go +++ b/namesys/interface.go @@ -34,8 +34,9 @@ import ( "time" context "context" - path "github.com/Casper-dev/Casper-server/path" ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" + + path "gitlab.com/casperDev/Casper-server/path" ) const ( diff --git a/namesys/ipns_select_test.go b/namesys/ipns_select_test.go index 7a3b259..6d7222f 100644 --- a/namesys/ipns_select_test.go +++ b/namesys/ipns_select_test.go @@ -8,10 +8,11 @@ import ( proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" - pb "github.com/Casper-dev/Casper-server/namesys/pb" - path "github.com/Casper-dev/Casper-server/path" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" + + pb "gitlab.com/casperDev/Casper-server/namesys/pb" + path "gitlab.com/casperDev/Casper-server/path" ) func shuffle(a []*pb.IpnsEntry) { diff --git a/namesys/namesys.go b/namesys/namesys.go index 6e45d6b..78d5c18 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -5,7 +5,7 @@ import ( "strings" "time" - path "github.com/Casper-dev/Casper-server/path" + path "gitlab.com/casperDev/Casper-server/path" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go index ddb26f4..6916035 100644 --- a/namesys/namesys_test.go +++ b/namesys/namesys_test.go @@ -6,9 +6,9 @@ import ( context "context" - path "github.com/Casper-dev/Casper-server/path" - offroute "github.com/Casper-dev/Casper-server/routing/offline" - "github.com/Casper-dev/Casper-server/unixfs" + path "gitlab.com/casperDev/Casper-server/path" + offroute "gitlab.com/casperDev/Casper-server/routing/offline" + "gitlab.com/casperDev/Casper-server/unixfs" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" dssync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" diff --git a/namesys/proquint.go b/namesys/proquint.go index 3b5a048..64d6321 100644 --- a/namesys/proquint.go +++ b/namesys/proquint.go @@ -4,8 +4,9 @@ import ( "errors" context "context" - path "github.com/Casper-dev/Casper-server/path" proquint "gx/ipfs/QmYnf27kzqR2cxt6LFZdrAFJuQd6785fTkBvMuEj9EeRxM/proquint" + + path "gitlab.com/casperDev/Casper-server/path" ) type ProquintResolver struct{} diff --git a/namesys/publisher.go b/namesys/publisher.go index 7b1354c..671c5e3 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -7,12 +7,12 @@ import ( "fmt" "time" - dag "github.com/Casper-dev/Casper-server/merkledag" - pb "github.com/Casper-dev/Casper-server/namesys/pb" - path "github.com/Casper-dev/Casper-server/path" - pin "github.com/Casper-dev/Casper-server/pin" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" - ft "github.com/Casper-dev/Casper-server/unixfs" + dag "gitlab.com/casperDev/Casper-server/merkledag" + pb "gitlab.com/casperDev/Casper-server/namesys/pb" + path "gitlab.com/casperDev/Casper-server/path" + pin "gitlab.com/casperDev/Casper-server/pin" + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" + ft "gitlab.com/casperDev/Casper-server/unixfs" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/namesys/publisher_test.go b/namesys/publisher_test.go index 6408911..87339e2 100644 --- a/namesys/publisher_test.go +++ b/namesys/publisher_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - path "github.com/Casper-dev/Casper-server/path" - mockrouting "github.com/Casper-dev/Casper-server/routing/mock" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" + path "gitlab.com/casperDev/Casper-server/path" + mockrouting "gitlab.com/casperDev/Casper-server/routing/mock" + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" dssync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" diff --git a/namesys/republisher/repub.go b/namesys/republisher/repub.go index 2d61938..8c15a1e 100644 --- a/namesys/republisher/repub.go +++ b/namesys/republisher/repub.go @@ -5,11 +5,11 @@ import ( "errors" "time" - keystore "github.com/Casper-dev/Casper-server/keystore" - namesys "github.com/Casper-dev/Casper-server/namesys" - pb "github.com/Casper-dev/Casper-server/namesys/pb" - path "github.com/Casper-dev/Casper-server/path" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" + keystore "gitlab.com/casperDev/Casper-server/keystore" + namesys "gitlab.com/casperDev/Casper-server/namesys" + pb "gitlab.com/casperDev/Casper-server/namesys/pb" + path "gitlab.com/casperDev/Casper-server/path" + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" goprocess "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go index 27e9246..dff7957 100644 --- a/namesys/republisher/repub_test.go +++ b/namesys/republisher/repub_test.go @@ -8,13 +8,14 @@ import ( context "context" goprocess "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" - "github.com/Casper-dev/Casper-server/core" - mock "github.com/Casper-dev/Casper-server/core/mock" - namesys "github.com/Casper-dev/Casper-server/namesys" - . "github.com/Casper-dev/Casper-server/namesys/republisher" - path "github.com/Casper-dev/Casper-server/path" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" mocknet "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/net/mock" + + "gitlab.com/casperDev/Casper-server/core" + mock "gitlab.com/casperDev/Casper-server/core/mock" + namesys "gitlab.com/casperDev/Casper-server/namesys" + . "gitlab.com/casperDev/Casper-server/namesys/republisher" + path "gitlab.com/casperDev/Casper-server/path" ) func TestRepublish(t *testing.T) { diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 0f42d77..8d8a0a7 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -6,10 +6,11 @@ import ( "testing" "time" - path "github.com/Casper-dev/Casper-server/path" - mockrouting "github.com/Casper-dev/Casper-server/routing/mock" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + path "gitlab.com/casperDev/Casper-server/path" + mockrouting "gitlab.com/casperDev/Casper-server/routing/mock" + ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" dssync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" diff --git a/namesys/routing.go b/namesys/routing.go index 8c84ee8..0ef95df 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -6,8 +6,8 @@ import ( "strings" "time" - pb "github.com/Casper-dev/Casper-server/namesys/pb" - path "github.com/Casper-dev/Casper-server/path" + pb "gitlab.com/casperDev/Casper-server/namesys/pb" + path "gitlab.com/casperDev/Casper-server/path" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" diff --git a/path/resolver.go b/path/resolver.go index bb8634c..6f06d20 100644 --- a/path/resolver.go +++ b/path/resolver.go @@ -7,7 +7,7 @@ import ( "fmt" "time" - dag "github.com/Casper-dev/Casper-server/merkledag" + dag "gitlab.com/casperDev/Casper-server/merkledag" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/path/resolver_test.go b/path/resolver_test.go index 78ba6da..c7e2501 100644 --- a/path/resolver_test.go +++ b/path/resolver_test.go @@ -5,9 +5,9 @@ import ( "fmt" "testing" - merkledag "github.com/Casper-dev/Casper-server/merkledag" - dagmock "github.com/Casper-dev/Casper-server/merkledag/test" - path "github.com/Casper-dev/Casper-server/path" + merkledag "gitlab.com/casperDev/Casper-server/merkledag" + dagmock "gitlab.com/casperDev/Casper-server/merkledag/test" + path "gitlab.com/casperDev/Casper-server/path" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" util "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/pin/gc/gc.go b/pin/gc/gc.go index 4b37730..f42c3eb 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - bstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - dag "github.com/Casper-dev/Casper-server/merkledag" - pin "github.com/Casper-dev/Casper-server/pin" + bstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + dag "gitlab.com/casperDev/Casper-server/merkledag" + pin "gitlab.com/casperDev/Casper-server/pin" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/pin/pin.go b/pin/pin.go index 8458a50..d545a72 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -9,8 +9,8 @@ import ( "sync" "time" - mdag "github.com/Casper-dev/Casper-server/merkledag" - dutils "github.com/Casper-dev/Casper-server/merkledag/utils" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + dutils "gitlab.com/casperDev/Casper-server/merkledag/utils" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/pin/pin_test.go b/pin/pin_test.go index 8b1b000..b43bfbb 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -5,10 +5,10 @@ import ( "testing" "time" - "github.com/Casper-dev/Casper-server/blocks/blockstore" - bs "github.com/Casper-dev/Casper-server/blockservice" - "github.com/Casper-dev/Casper-server/exchange/offline" - mdag "github.com/Casper-dev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bs "gitlab.com/casperDev/Casper-server/blockservice" + "gitlab.com/casperDev/Casper-server/exchange/offline" + mdag "gitlab.com/casperDev/Casper-server/merkledag" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/pin/set.go b/pin/set.go index 4968e14..d8ac5d2 100644 --- a/pin/set.go +++ b/pin/set.go @@ -9,8 +9,8 @@ import ( "hash/fnv" "sort" - "github.com/Casper-dev/Casper-server/merkledag" - "github.com/Casper-dev/Casper-server/pin/internal/pb" + "gitlab.com/casperDev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/pin/internal/pb" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/pin/set_test.go b/pin/set_test.go index be91114..f3cc26a 100644 --- a/pin/set_test.go +++ b/pin/set_test.go @@ -5,10 +5,10 @@ import ( "encoding/binary" "testing" - blockstore "github.com/Casper-dev/Casper-server/blocks/blockstore" - bserv "github.com/Casper-dev/Casper-server/blockservice" - offline "github.com/Casper-dev/Casper-server/exchange/offline" - dag "github.com/Casper-dev/Casper-server/merkledag" + blockstore "gitlab.com/casperDev/Casper-server/blocks/blockstore" + bserv "gitlab.com/casperDev/Casper-server/blockservice" + offline "gitlab.com/casperDev/Casper-server/exchange/offline" + dag "gitlab.com/casperDev/Casper-server/merkledag" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" diff --git a/plugin/ipld.go b/plugin/ipld.go index 5da644a..f93966e 100644 --- a/plugin/ipld.go +++ b/plugin/ipld.go @@ -1,7 +1,7 @@ package plugin import ( - "github.com/Casper-dev/Casper-server/core/coredag" + "gitlab.com/casperDev/Casper-server/core/coredag" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/plugin/loader/initializer.go b/plugin/loader/initializer.go index 75e5298..4ce4820 100644 --- a/plugin/loader/initializer.go +++ b/plugin/loader/initializer.go @@ -1,8 +1,8 @@ package loader import ( - "github.com/Casper-dev/Casper-server/core/coredag" - "github.com/Casper-dev/Casper-server/plugin" + "gitlab.com/casperDev/Casper-server/core/coredag" + "gitlab.com/casperDev/Casper-server/plugin" format "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/plugin/loader/load.go b/plugin/loader/load.go index edeae70..2cb8ae5 100644 --- a/plugin/loader/load.go +++ b/plugin/loader/load.go @@ -4,7 +4,7 @@ import ( "fmt" "os" - "github.com/Casper-dev/Casper-server/plugin" + "gitlab.com/casperDev/Casper-server/plugin" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" ) diff --git a/plugin/loader/load_linux.go b/plugin/loader/load_linux.go index 25300e1..bc5d811 100644 --- a/plugin/loader/load_linux.go +++ b/plugin/loader/load_linux.go @@ -7,7 +7,7 @@ import ( "path/filepath" "plugin" - iplugin "github.com/Casper-dev/Casper-server/plugin" + iplugin "gitlab.com/casperDev/Casper-server/plugin" ) func init() { diff --git a/plugin/loader/preload.go b/plugin/loader/preload.go index d6ae46a..fed1b52 100644 --- a/plugin/loader/preload.go +++ b/plugin/loader/preload.go @@ -1,7 +1,7 @@ package loader import ( - "github.com/Casper-dev/Casper-server/plugin" + "gitlab.com/casperDev/Casper-server/plugin" ) // DO NOT EDIT THIS FILE diff --git a/plugin/plugins/git/git.go b/plugin/plugins/git/git.go index 26fcc7a..7c9bc04 100644 --- a/plugin/plugins/git/git.go +++ b/plugin/plugins/git/git.go @@ -6,8 +6,8 @@ import ( "io" "math" - "github.com/Casper-dev/Casper-server/core/coredag" - "github.com/Casper-dev/Casper-server/plugin" + "gitlab.com/casperDev/Casper-server/core/coredag" + "gitlab.com/casperDev/Casper-server/plugin" "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/repo/config/casper.go b/repo/config/casper.go index db1fc67..8f4f934 100644 --- a/repo/config/casper.go +++ b/repo/config/casper.go @@ -1,6 +1,8 @@ package config type Casper struct { - IPAddress string - RepoSize int64 + DiskSizeBytes int64 + IPAddress string + TelegramAddress string + ConnectionPort string } diff --git a/repo/config/config.go b/repo/config/config.go index ba927b4..93b325b 100644 --- a/repo/config/config.go +++ b/repo/config/config.go @@ -9,7 +9,7 @@ import ( "path/filepath" "strings" - "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/mitchellh/go-homedir" + "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/mitchellh/go-homedir" ) // Config is used to load ipfs config files. @@ -24,7 +24,7 @@ type Config struct { Gateway Gateway // local node's gateway server options API API // local node's API settings Swarm SwarmConfig - Casper Casper + Casper Casper Reprovider Reprovider Experimental Experiments @@ -38,7 +38,7 @@ const ( // DefaultConfigFile is the filename of the configuration file DefaultConfigFile = "config" // EnvDir is the environment variable used to change the path root. - EnvDir = "CASPER_SERVER_PATH" + EnvDir = "CASPER_REPO_PATH" ) // PathRoot returns the default configuration root directory diff --git a/repo/config/init.go b/repo/config/init.go index d3b414e..9c66b9d 100644 --- a/repo/config/init.go +++ b/repo/config/init.go @@ -7,9 +7,8 @@ import ( "io" "time" - peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" + "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" ci "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto" - "github.com/Casper-dev/Casper-SC/casper_sc" ) func Init(out io.Writer, nBitsForKeypair int) (*Config, error) { @@ -26,7 +25,6 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) { datastore := DefaultDatastoreConfig() conf := &Config{ - // setup the node's default addresses. // NOTE: two swarm listen addrs, one tcp, one utp. Addresses: Addresses{ @@ -80,10 +78,17 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) { GracePeriod: DefaultConnMgrGracePeriod.String(), Type: "basic", }, + NAT: NATOpts{ + TraversalSC: false, + StunServers: []string{}, + TurnServers: []TurnServer{}, + }, }, Casper: Casper{ - IPAddress: Casper_SC.GetIPv4(), - RepoSize: DefaultCasperDiskSize, /// ceil(25.6 * 1024^3) + DiskSizeBytes: DefaultCasperDiskSize, + TelegramAddress: DefaultCasperTelegramAddress, + IPAddress: DefaultCasperConnectionIP, + ConnectionPort: DefaultCasperConnectionPort, }, } @@ -102,12 +107,17 @@ const DefaultConnMgrLowWater = 600 // grace period const DefaultConnMgrGracePeriod = time.Second * 20 -const DefaultCasperDiskSize = int64(27487790695) /// ceil(25.6 * 1024^3) +const DefaultCasperDiskSize = int64(27487790694) + +const DefaultCasperTelegramAddress = " " + +const DefaultCasperConnectionIP = "0.0.0.0" +const DefaultCasperConnectionPort = "9090" // DefaultDatastoreConfig is an internal function exported to aid in testing. func DefaultDatastoreConfig() Datastore { return Datastore{ - StorageMax: "25GB", + StorageMax: "10GB", StorageGCWatermark: 90, // 90% GCPeriod: "1h", BloomFilterSize: 0, diff --git a/repo/config/profile.go b/repo/config/profile.go index 3a7bf36..b799f4d 100644 --- a/repo/config/profile.go +++ b/repo/config/profile.go @@ -2,6 +2,26 @@ package config // ConfigProfiles is a map holding configuration transformers var ConfigProfiles = map[string]func(*Config) error{ + "nat": func(c *Config) error { + defaultTurnServers := []TurnServer{ + TurnServer{ // default server for testing purposes + Address: "/ip4/195.201.26.73/tcp/3478", + User: "Casper", + Password: "TestPassword", + }, + } + defaultStunServers := []string{"195.201.26.73:3478"} + + c.Swarm.NAT.TraversalSC = true + c.Swarm.NAT.TurnServers = defaultTurnServers + c.Swarm.NAT.StunServers = defaultStunServers + return nil + }, + "relay": func(c *Config) error { + c.Swarm.DisableRelay = false + c.Swarm.EnableRelayHop = true + return nil + }, "server": func(c *Config) error { // defaultServerFilters has a list of non-routable IPv4 prefixes diff --git a/repo/config/swarm.go b/repo/config/swarm.go index 5b128f3..df0e96a 100644 --- a/repo/config/swarm.go +++ b/repo/config/swarm.go @@ -6,6 +6,7 @@ type SwarmConfig struct { DisableNatPortMap bool DisableRelay bool EnableRelayHop bool + NAT NATOpts // STUN/TURN server addresses ConnMgr ConnMgr } @@ -17,3 +18,18 @@ type ConnMgr struct { HighWater int GracePeriod string } + +// NATOpts defines configuration option for IP determination +// if TraversalSC is false, one of local IPs will be sent to SC +type NATOpts struct { + TraversalSC bool + StunServers []string `json:",omitempty"` + // TODO add use credentials + TurnServers []TurnServer `json:",omitempty"` +} + +type TurnServer struct { + Address string + User string + Password string +} diff --git a/repo/fsrepo/config_test.go b/repo/fsrepo/config_test.go index c8a697a..c44ee75 100644 --- a/repo/fsrepo/config_test.go +++ b/repo/fsrepo/config_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - config "github.com/Casper-dev/Casper-server/repo/config" + config "gitlab.com/casperDev/Casper-server/repo/config" ) // note: to test sorting of the mountpoints in the disk spec they are diff --git a/repo/fsrepo/datastores.go b/repo/fsrepo/datastores.go index 8e139b2..68db648 100644 --- a/repo/fsrepo/datastores.go +++ b/repo/fsrepo/datastores.go @@ -8,7 +8,7 @@ import ( "path/filepath" "sort" - repo "github.com/Casper-dev/Casper-server/repo" + repo "gitlab.com/casperDev/Casper-server/repo" measure "gx/ipfs/QmSb95iHExSSb47zpmyn5CyY5PZidVWSjyKyDqgYQrnKor/go-ds-measure" flatfs "gx/ipfs/QmUTshC2PP4ZDqkrFfDU4JGJFMWjYnunxPgkQ6ZCA2hGqh/go-ds-flatfs" diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index dc86353..e77e24f 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -11,17 +11,17 @@ import ( "strings" "sync" - filestore "github.com/Casper-dev/Casper-server/filestore" - keystore "github.com/Casper-dev/Casper-server/keystore" - repo "github.com/Casper-dev/Casper-server/repo" - "github.com/Casper-dev/Casper-server/repo/common" - config "github.com/Casper-dev/Casper-server/repo/config" - lockfile "github.com/Casper-dev/Casper-server/repo/fsrepo/lock" - mfsr "github.com/Casper-dev/Casper-server/repo/fsrepo/migrations" - serialize "github.com/Casper-dev/Casper-server/repo/fsrepo/serialize" - dir "github.com/Casper-dev/Casper-server/thirdparty/dir" - - "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/mitchellh/go-homedir" + filestore "gitlab.com/casperDev/Casper-server/filestore" + keystore "gitlab.com/casperDev/Casper-server/keystore" + repo "gitlab.com/casperDev/Casper-server/repo" + "gitlab.com/casperDev/Casper-server/repo/common" + config "gitlab.com/casperDev/Casper-server/repo/config" + lockfile "gitlab.com/casperDev/Casper-server/repo/fsrepo/lock" + mfsr "gitlab.com/casperDev/Casper-server/repo/fsrepo/migrations" + serialize "gitlab.com/casperDev/Casper-server/repo/fsrepo/serialize" + dir "gitlab.com/casperDev/Casper-server/thirdparty/dir" + + "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/mitchellh/go-homedir" util "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" measure "gx/ipfs/QmSb95iHExSSb47zpmyn5CyY5PZidVWSjyKyDqgYQrnKor/go-ds-measure" diff --git a/repo/fsrepo/fsrepo_test.go b/repo/fsrepo/fsrepo_test.go index 16e3263..673a9e9 100644 --- a/repo/fsrepo/fsrepo_test.go +++ b/repo/fsrepo/fsrepo_test.go @@ -7,9 +7,10 @@ import ( "path/filepath" "testing" - "github.com/Casper-dev/Casper-server/repo/config" - "github.com/Casper-dev/Casper-server/thirdparty/assert" datastore "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" + + "gitlab.com/casperDev/Casper-server/repo/config" + "gitlab.com/casperDev/Casper-server/thirdparty/assert" ) // swap arg order diff --git a/repo/fsrepo/migrations/mfsr_test.go b/repo/fsrepo/migrations/mfsr_test.go index 3f0e5f5..9ca6b99 100644 --- a/repo/fsrepo/migrations/mfsr_test.go +++ b/repo/fsrepo/migrations/mfsr_test.go @@ -6,7 +6,7 @@ import ( "strconv" "testing" - "github.com/Casper-dev/Casper-server/thirdparty/assert" + "gitlab.com/casperDev/Casper-server/thirdparty/assert" ) func testVersionFile(v string, t *testing.T) (rp RepoPath) { diff --git a/repo/fsrepo/misc.go b/repo/fsrepo/misc.go index c0afa59..ec9f9ec 100644 --- a/repo/fsrepo/misc.go +++ b/repo/fsrepo/misc.go @@ -3,8 +3,8 @@ package fsrepo import ( "os" - homedir "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/mitchellh/go-homedir" - "github.com/Casper-dev/Casper-server/repo/config" + homedir "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/mitchellh/go-homedir" + "gitlab.com/casperDev/Casper-server/repo/config" ) // BestKnownPath returns the best known fsrepo path. If the ENV override is diff --git a/repo/fsrepo/serialize/serialize.go b/repo/fsrepo/serialize/serialize.go index 59b20ba..dd86e6b 100644 --- a/repo/fsrepo/serialize/serialize.go +++ b/repo/fsrepo/serialize/serialize.go @@ -8,9 +8,10 @@ import ( "os" "path/filepath" - "github.com/Casper-dev/Casper-server/repo/config" "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" "gx/ipfs/QmdYwCmx8pZRkzdcd8MhmLJqYVoVTC1aGsy5Q4reMGLNLg/atomicfile" + + "gitlab.com/casperDev/Casper-server/repo/config" ) // ReadConfigFile reads the config from `filename` into `cfg`. diff --git a/repo/fsrepo/serialize/serialize_test.go b/repo/fsrepo/serialize/serialize_test.go index 61994f7..2728a95 100644 --- a/repo/fsrepo/serialize/serialize_test.go +++ b/repo/fsrepo/serialize/serialize_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - config "github.com/Casper-dev/Casper-server/repo/config" + config "gitlab.com/casperDev/Casper-server/repo/config" ) func TestConfig(t *testing.T) { diff --git a/repo/mock.go b/repo/mock.go index e7555a3..fb7a4b5 100644 --- a/repo/mock.go +++ b/repo/mock.go @@ -3,9 +3,9 @@ package repo import ( "errors" - filestore "github.com/Casper-dev/Casper-server/filestore" - keystore "github.com/Casper-dev/Casper-server/keystore" - "github.com/Casper-dev/Casper-server/repo/config" + filestore "gitlab.com/casperDev/Casper-server/filestore" + keystore "gitlab.com/casperDev/Casper-server/keystore" + "gitlab.com/casperDev/Casper-server/repo/config" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" ) diff --git a/repo/repo.go b/repo/repo.go index 210f018..764ce98 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -4,9 +4,9 @@ import ( "errors" "io" - filestore "github.com/Casper-dev/Casper-server/filestore" - keystore "github.com/Casper-dev/Casper-server/keystore" - config "github.com/Casper-dev/Casper-server/repo/config" + filestore "gitlab.com/casperDev/Casper-server/filestore" + keystore "gitlab.com/casperDev/Casper-server/keystore" + config "gitlab.com/casperDev/Casper-server/repo/config" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr" diff --git a/routing/mock/centralized_client.go b/routing/mock/centralized_client.go index 02c5d0a..345dc7a 100644 --- a/routing/mock/centralized_client.go +++ b/routing/mock/centralized_client.go @@ -5,9 +5,10 @@ import ( "errors" "time" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" diff --git a/routing/mock/centralized_test.go b/routing/mock/centralized_test.go index c3d746b..5584285 100644 --- a/routing/mock/centralized_test.go +++ b/routing/mock/centralized_test.go @@ -5,9 +5,10 @@ import ( "testing" "time" - delay "github.com/Casper-dev/Casper-server/thirdparty/delay" "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + delay "gitlab.com/casperDev/Casper-server/thirdparty/delay" + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" diff --git a/routing/mock/interface.go b/routing/mock/interface.go index 4d6050d..c8dfe2f 100644 --- a/routing/mock/interface.go +++ b/routing/mock/interface.go @@ -7,9 +7,10 @@ package mockrouting import ( "context" - delay "github.com/Casper-dev/Casper-server/thirdparty/delay" "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + delay "gitlab.com/casperDev/Casper-server/thirdparty/delay" + routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer" diff --git a/routing/none/none_client.go b/routing/none/none_client.go index 0306575..a7a10eb 100644 --- a/routing/none/none_client.go +++ b/routing/none/none_client.go @@ -4,7 +4,7 @@ import ( "context" "errors" - repo "github.com/Casper-dev/Casper-server/repo" + repo "gitlab.com/casperDev/Casper-server/repo" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" diff --git a/routing/offline/offline.go b/routing/offline/offline.go index acea5c7..7ecc37a 100644 --- a/routing/offline/offline.go +++ b/routing/offline/offline.go @@ -5,7 +5,7 @@ import ( "errors" "time" - dshelp "github.com/Casper-dev/Casper-server/thirdparty/ds-help" + dshelp "gitlab.com/casperDev/Casper-server/thirdparty/ds-help" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" routing "gx/ipfs/QmPR2JzfKd9poHx9XBhzoFeBBC31ZM3W5iUPKJZWyaoZZm/go-libp2p-routing" diff --git a/submodule/go-ipld-cbor/node.go b/submodule/go-ipld-cbor/node.go new file mode 100644 index 0000000..0ac6b85 --- /dev/null +++ b/submodule/go-ipld-cbor/node.go @@ -0,0 +1,573 @@ +package cbornode + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "strconv" + "strings" + + cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" + blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" + + cbor "github.com/whyrusleeping/cbor/go" + bl "gitlab.com/casperDev/Casper-server/blocks" +) + +const CBORTagLink = 42 + +var log = logging.Logger("cbornode") + +// Decode a CBOR encoded Block into an IPLD Node. +// +// This method *does not* canonicalize and *will* preserve the CID. As a matter +// of fact, it will assume that `block.Cid()` returns the correct CID and will +// make no effort to validate this assumption. +// +// In general, you should not be calling this method directly. Instead, you +// should be calling the `Decode` method from the `go-ipld-format` package. That +// method will pick the right decoder based on the Block's CID. +// +// Note: This function keeps a reference to `block` and assumes that it is +// immutable. +func DecodeBlock(block blocks.Block) (node.Node, error) { + return decodeBlock(block) +} + +func decodeBlock(block blocks.Block) (*Node, error) { + log.Debugf("CBOR %x", block.RawData()) + _, d := bl.SplitData(block.RawData()) + m, err := decodeCBOR(d) + + if err != nil { + return nil, err + } + tree, err := compTree(m) + if err != nil { + return nil, err + } + links, err := compLinks(m) + if err != nil { + return nil, err + } + return &Node{ + obj: m, + tree: tree, + links: links, + raw: block.RawData(), + cid: block.Cid(), + }, nil +} + +var _ node.DecodeBlockFunc = DecodeBlock + +// Decode a CBOR object into an IPLD Node. +// +// If passed a non-canonical CBOR node, this function will canonicalize it. +// Therefore, `bytes.Equal(b, Decode(b).RawData())` may not hold. If you already +// have a CID for this data and want to ensure that it doesn't change, you +// should use `DecodeBlock`. +// mhType is multihash code to use for hashing, for example mh.SHA2_256 +// +// Note: This function does not hold onto `b`. You may reuse it. +func Decode(b []byte, mhType uint64, mhLen int) (*Node, error) { + m, err := decodeCBOR(b) + if err != nil { + return nil, err + } + // We throw away `b` here to ensure that we canonicalize the encoded + // CBOR object. + return WrapObject(m, mhType, mhLen) +} + +// DecodeInto decodes a serialized ipld cbor object into the given object. +func DecodeInto(b []byte, v interface{}) error { + // The cbor library really doesnt make this sort of operation easy on us + m, err := decodeCBOR(b) + if err != nil { + return err + } + + jsonable, err := convertToJsonIsh(m) + if err != nil { + return err + } + + jsonb, err := json.Marshal(jsonable) + if err != nil { + return err + } + + return json.Unmarshal(jsonb, v) + +} + +// Decodes a cbor node into an object. +func decodeCBOR(b []byte) (m interface{}, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("cbor panic: %s", r) + } + }() + dec := cbor.NewDecoder(bytes.NewReader(b)) + dec.TagDecoders[CBORTagLink] = new(IpldLinkDecoder) + err = dec.Decode(&m) + return +} + +var ErrNoSuchLink = errors.New("no such link found") + +type Node struct { + obj interface{} + tree []string + links []*node.Link + raw []byte + cid *cid.Cid +} + +func WrapObject(m interface{}, mhType uint64, mhLen int) (*Node, error) { + data, err := DumpObject(m) + if err != nil { + return nil, err + } + if mhType == math.MaxUint64 { + mhType = mh.SHA2_256 + } + + hash, err := mh.Sum(data, mhType, mhLen) + if err != nil { + return nil, err + } + c := cid.NewCidV1(cid.DagCBOR, hash) + + block, err := blocks.NewBlockWithCid(data, c) + if err != nil { + // TODO: Shouldn't this just panic? + return nil, err + } + // Do not reuse `m`. We need to re-decode it to put it in the right + // form. + return decodeBlock(block) +} + +func (n *Node) Resolve(path []string) (interface{}, []string, error) { + var cur interface{} = n.obj + for i, val := range path { + switch curv := cur.(type) { + case map[interface{}]interface{}: + next, ok := curv[val] + if !ok { + return nil, nil, ErrNoSuchLink + } + + cur = next + case []interface{}: + n, err := strconv.Atoi(val) + if err != nil { + return nil, nil, err + } + + if n < 0 || n >= len(curv) { + return nil, nil, fmt.Errorf("array index out of range") + } + + cur = curv[n] + case *cid.Cid: + return &node.Link{Cid: curv}, path[i:], nil + default: + return nil, nil, errors.New("tried to resolve through object that had no links") + } + } + + lnk, ok := cur.(*cid.Cid) + if ok { + return &node.Link{Cid: lnk}, nil, nil + } + + jsonish, err := convertToJsonIsh(cur) + if err != nil { + return nil, nil, err + } + + return jsonish, nil, nil +} + +func (n *Node) Copy() node.Node { + links := make([]*node.Link, len(n.links)) + copy(links, n.links) + + raw := make([]byte, len(n.raw)) + copy(raw, n.raw) + + tree := make([]string, len(n.tree)) + copy(tree, n.tree) + + return &Node{ + obj: copyObj(n.obj), + links: links, + raw: raw, + tree: tree, + cid: n.cid, + } +} + +func copyObj(i interface{}) interface{} { + switch i := i.(type) { + case map[interface{}]interface{}: + out := make(map[interface{}]interface{}) + for k, v := range i { + out[k] = copyObj(v) + } + return out + case []interface{}: + var out []interface{} + for _, v := range i { + out = append(out, copyObj(v)) + } + return out + default: + // being lazy for now + // use caution + return i + } +} + +func (n *Node) ResolveLink(path []string) (*node.Link, []string, error) { + obj, rest, err := n.Resolve(path) + if err != nil { + return nil, nil, err + } + + lnk, ok := obj.(*node.Link) + if ok { + return lnk, rest, nil + } + + return nil, rest, fmt.Errorf("found non-link at given path") +} + +func linkCast(lnk interface{}) (*node.Link, error) { + lnkb, ok := lnk.([]byte) + if !ok { + return nil, errors.New("incorrectly formatted link") + } + + c, err := cid.Cast(lnkb) + if err != nil { + return nil, err + } + + return &node.Link{Cid: c}, nil +} + +func (n *Node) Tree(path string, depth int) []string { + if path == "" && depth == -1 { + return n.tree + } + + var out []string + for _, t := range n.tree { + if !strings.HasPrefix(t, path) { + continue + } + + sub := strings.TrimLeft(t[len(path):], "/") + if sub == "" { + continue + } + + if depth < 0 { + out = append(out, sub) + continue + } + + parts := strings.Split(sub, "/") + if len(parts) <= depth { + out = append(out, sub) + } + } + return out +} + +func compTree(obj interface{}) ([]string, error) { + var out []string + err := traverse(obj, "", func(name string, val interface{}) error { + if name != "" { + out = append(out, name[1:]) + } + return nil + }) + if err != nil { + return nil, err + } + + return out, nil +} + +func (n *Node) Links() []*node.Link { + return n.links +} + +func compLinks(obj interface{}) ([]*node.Link, error) { + var out []*node.Link + err := traverse(obj, "", func(name string, val interface{}) error { + if lnk, ok := val.(*cid.Cid); ok { + out = append(out, &node.Link{Cid: lnk}) + } + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +func traverse(obj interface{}, cur string, cb func(string, interface{}) error) error { + if err := cb(cur, obj); err != nil { + return err + } + + switch obj := obj.(type) { + case map[interface{}]interface{}: + for k, v := range obj { + ks, ok := k.(string) + if !ok { + return errors.New("map key was not a string") + } + this := cur + "/" + ks + if err := traverse(v, this, cb); err != nil { + return err + } + } + return nil + case []interface{}: + for i, v := range obj { + this := fmt.Sprintf("%s/%d", cur, i) + if err := traverse(v, this, cb); err != nil { + return err + } + } + return nil + default: + return nil + } +} + +func (n *Node) RawData() []byte { + return n.raw +} + +func DumpObject(obj interface{}) ([]byte, error) { + buf := new(bytes.Buffer) + enc := cbor.NewEncoder(buf) + enc.SetFilter(EncoderFilter) + err := enc.Encode(obj) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (n *Node) Cid() *cid.Cid { + return n.cid +} + +func (n *Node) Loggable() map[string]interface{} { + return map[string]interface{}{ + "node_type": "cbor", + "cid": n.Cid(), + } +} + +func (n *Node) Size() (uint64, error) { + return uint64(len(n.RawData())), nil +} + +func (n *Node) Stat() (*node.NodeStat, error) { + return &node.NodeStat{}, nil +} + +func (n *Node) String() string { + return n.Cid().String() +} + +func (n *Node) MarshalJSON() ([]byte, error) { + out, err := convertToJsonIsh(n.obj) + if err != nil { + return nil, err + } + + return json.Marshal(out) +} + +func toSaneMap(n map[interface{}]interface{}) (interface{}, error) { + if lnk, ok := n["/"]; ok && len(n) == 1 { + lnkb, ok := lnk.([]byte) + if !ok { + return nil, fmt.Errorf("link value should have been bytes") + } + + c, err := cid.Cast(lnkb) + if err != nil { + return nil, err + } + + return map[string]interface{}{"/": c}, nil + } + out := make(map[string]interface{}) + for k, v := range n { + ks, ok := k.(string) + if !ok { + return nil, fmt.Errorf("map keys must be strings") + } + + obj, err := convertToJsonIsh(v) + if err != nil { + return nil, err + } + + out[ks] = obj + } + + return out, nil +} + +func convertToJsonIsh(v interface{}) (interface{}, error) { + switch v := v.(type) { + case map[interface{}]interface{}: + return toSaneMap(v) + case []interface{}: + var out []interface{} + if len(v) == 0 && v != nil { + return []interface{}{}, nil + } + for _, i := range v { + obj, err := convertToJsonIsh(i) + if err != nil { + return nil, err + } + + out = append(out, obj) + } + return out, nil + default: + return v, nil + } +} + +func FromJson(r io.Reader, mhType uint64, mhLen int) (*Node, error) { + var m interface{} + err := json.NewDecoder(r).Decode(&m) + if err != nil { + return nil, err + } + + obj, err := convertToCborIshObj(m) + if err != nil { + return nil, err + } + + return WrapObject(obj, mhType, mhLen) +} + +func convertMapSIToCbor(from map[string]interface{}) (map[interface{}]interface{}, error) { + to := make(map[interface{}]interface{}) + for k, v := range from { + out, err := convertToCborIshObj(v) + if err != nil { + return nil, err + } + to[k] = out + } + + return to, nil +} + +func convertToCborIshObj(i interface{}) (interface{}, error) { + switch v := i.(type) { + case map[string]interface{}: + if lnk, ok := v["/"]; ok && len(v) == 1 { + // special case for links + vstr, ok := lnk.(string) + if !ok { + return nil, fmt.Errorf("link should have been a string") + } + + return cid.Decode(vstr) + } + + return convertMapSIToCbor(v) + case []interface{}: + var out []interface{} + for _, o := range v { + obj, err := convertToCborIshObj(o) + if err != nil { + return nil, err + } + + out = append(out, obj) + } + + return out, nil + default: + return v, nil + } +} + +func EncoderFilter(i interface{}) interface{} { + link, ok := i.(*cid.Cid) + if !ok { + return i + } + + return &cbor.CBORTag{ + Tag: CBORTagLink, + WrappedObject: append([]byte{0}, link.Bytes()...), // TODO: manually doing binary multibase + } +} + +type IpldLinkDecoder struct{} + +func (d *IpldLinkDecoder) DecodeTarget() interface{} { + return &[]byte{} +} + +func (d *IpldLinkDecoder) GetTag() uint64 { + return CBORTagLink +} + +func (d *IpldLinkDecoder) PostDecode(i interface{}) (interface{}, error) { + ibarr, ok := i.(*[]byte) + if !ok { + return nil, fmt.Errorf("expected a byte array in IpldLink PostDecode") + } + + barr := *ibarr + + if len(barr) == 0 { + return nil, fmt.Errorf("link value was empty") + } + + // TODO: manually doing multibase checking here since our deps don't + // support binary multibase yet + if barr[0] != 0 { + return nil, fmt.Errorf("invalid multibase on ipld link") + } + + c, err := cid.Cast(barr[1:]) + if err != nil { + return nil, err + } + + return c, nil +} + +var _ cbor.TagDecoder = (*IpldLinkDecoder)(nil) +var _ node.Node = (*Node)(nil) diff --git a/submodule/go-ipld-cbor/node_test.go b/submodule/go-ipld-cbor/node_test.go new file mode 100644 index 0000000..3d09ce6 --- /dev/null +++ b/submodule/go-ipld-cbor/node_test.go @@ -0,0 +1,433 @@ +package cbornode + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "sort" + "strings" + "testing" + + blocks "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" + u "github.com/ipfs/go-ipfs-util" + mh "github.com/multiformats/go-multihash" +) + +func assertCid(c *cid.Cid, exp string) error { + if c.String() != exp { + return fmt.Errorf("expected cid of %s, got %s", exp, c) + } + return nil +} + +func TestNonObject(t *testing.T) { + nd, err := WrapObject("", mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + if err := assertCid(nd.Cid(), "zdpuAuvdvGBYa3apsrf63GU9RZcrf5EBwvb82pHjUTyecbvD8"); err != nil { + t.Fatal(err) + } + + back, err := Decode(nd.Copy().RawData(), mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + if err := assertCid(back.Cid(), "zdpuAuvdvGBYa3apsrf63GU9RZcrf5EBwvb82pHjUTyecbvD8"); err != nil { + t.Fatal(err) + } +} + +func TestDecodeInto(t *testing.T) { + nd, err := WrapObject(map[string]string{ + "name": "foo", + }, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + var m map[string]string + err = DecodeInto(nd.RawData(), &m) + if err != nil { + t.Fatal(err) + } + if len(m) != 1 || m["name"] != "foo" { + t.Fatal("failed to decode object") + } +} + +func TestDecodeIntoNonObject(t *testing.T) { + nd, err := WrapObject("foobar", mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + var s string + err = DecodeInto(nd.RawData(), &s) + if err != nil { + t.Fatal(err) + } + if s != "foobar" { + t.Fatal("strings don't match") + } +} + +func TestBasicMarshal(t *testing.T) { + c := cid.NewCidV0(u.Hash([]byte("something"))) + + obj := map[string]interface{}{ + "name": "foo", + "bar": c, + } + + nd, err := WrapObject(obj, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + if err := assertCid(nd.Cid(), "zdpuApUZEHofKXuTs2Yv2CLBeiASQrc9FojFLSZWcyZq6dZhb"); err != nil { + t.Fatal(err) + } + + back, err := Decode(nd.RawData(), mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + if err := assertCid(back.Cid(), "zdpuApUZEHofKXuTs2Yv2CLBeiASQrc9FojFLSZWcyZq6dZhb"); err != nil { + t.Fatal(err) + } + + lnk, _, err := back.ResolveLink([]string{"bar"}) + if err != nil { + t.Fatal(err) + } + + if !lnk.Cid.Equals(c) { + t.Fatal("expected cid to match") + } + + if !nd.Cid().Equals(back.Cid()) { + t.Fatal("re-serialize failed to generate same cid") + } +} + +func TestMarshalRoundtrip(t *testing.T) { + c1 := cid.NewCidV0(u.Hash([]byte("something1"))) + c2 := cid.NewCidV0(u.Hash([]byte("something2"))) + c3 := cid.NewCidV0(u.Hash([]byte("something3"))) + + obj := map[interface{}]interface{}{ + "foo": "bar", + "baz": []interface{}{ + c1, + c2, + }, + "cats": map[string]interface{}{ + "qux": c3, + }, + } + + nd1, err := WrapObject(obj, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + if err := assertCid(nd1.Cid(), "zdpuAwc5bPhfHGdA4rs3qKzr3QB3Fjp3D16C8BRRyWzTPpY9R"); err != nil { + t.Fatal(err) + } + + if len(nd1.Links()) != 3 { + t.Fatal("didnt have enough links") + } + + nd2, err := Decode(nd1.RawData(), mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + if !nd1.Cid().Equals(nd2.Cid()) { + t.Fatal("objects didnt match between marshalings") + } + + lnk, rest, err := nd2.ResolveLink([]string{"baz", "1", "bop"}) + if err != nil { + t.Fatal(err) + } + + if !lnk.Cid.Equals(c2) { + t.Fatal("expected c2") + } + + if len(rest) != 1 || rest[0] != "bop" { + t.Fatal("should have had one path element remaning after resolve") + } + + out, err := nd1.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + t.Log(string(out)) +} + +func assertStringsEqual(t *testing.T, a, b []string) { + if len(a) != len(b) { + t.Fatal("lengths differed: ", a, b) + } + + sort.Strings(a) + sort.Strings(b) + + for i, v := range a { + if v != b[i] { + t.Fatal("got mismatch: ", a, b) + } + } +} + +func TestTree(t *testing.T) { + c1 := cid.NewCidV0(u.Hash([]byte("something1"))) + c2 := cid.NewCidV0(u.Hash([]byte("something2"))) + c3 := cid.NewCidV0(u.Hash([]byte("something3"))) + c4 := cid.NewCidV0(u.Hash([]byte("something4"))) + + obj := map[string]interface{}{ + "foo": c1, + "baz": []interface{}{c2, c3, "c"}, + "cats": map[interface{}]interface{}{ + "qux": map[string]interface{}{ + "boo": 1, + "baa": c4, + "bee": 3, + "bii": 4, + "buu": map[interface{}]string{ + "coat": "rain", + }, + }, + }, + } + + nd, err := WrapObject(obj, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + if err := assertCid(nd.Cid(), "zdpuAqobkonFx9i79VEDiz2WcU2dC1YU8ApEVRwSC8sx5cjUP"); err != nil { + t.Fatal(err) + } + + full := []string{ + "foo", + "baz", + "baz/0", + "baz/1", + "baz/2", + "cats", + "cats/qux", + "cats/qux/boo", + "cats/qux/baa", + "cats/qux/bee", + "cats/qux/bii", + "cats/qux/buu", + "cats/qux/buu/coat", + } + + assertStringsEqual(t, full, nd.Tree("", -1)) + + cats := []string{ + "qux", + "qux/boo", + "qux/baa", + "qux/bee", + "qux/bii", + "qux/buu", + "qux/buu/coat", + } + + assertStringsEqual(t, cats, nd.Tree("cats", -1)) + + toplevel := []string{ + "foo", + "baz", + "cats", + } + + assertStringsEqual(t, toplevel, nd.Tree("", 1)) + assertStringsEqual(t, []string{}, nd.Tree("", 0)) +} + +func TestParsing(t *testing.T) { + b := []byte("\xd9\x01\x02\x58\x25\xa5\x03\x22\x12\x20\x65\x96\x50\xfc\x34\x43\xc9\x16\x42\x80\x48\xef\xc5\xba\x45\x58\xdc\x86\x35\x94\x98\x0a\x59\xf5\xcb\x3c\x4d\x84\x86\x7e\x6d\x31") + + n, err := Decode(b, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + if err := assertCid(n.Cid(), "zdpuApUpLTkn3YYUeuMjWToYn8nt4KQY9kQqd9uL6vwHxXnQN"); err != nil { + t.Fatal(err) + } + + n, err = Decode(b, mh.SHA2_512, -1) + if err != nil { + t.Fatal(err) + } + + if err := assertCid(n.Cid(), "zBwW8WMJocqnuegmghC9MyTw26Ywsdp8KTPUKkhNrnefa1X3RoNtvCCJ6kLbur2bS6TNriRb5SyFKLq9jpwtra9Fsxdd9"); err != nil { + t.Fatal(err) + } +} + +func TestFromJson(t *testing.T) { + data := `{ + "something": {"/":"zb2rhisguzLFRJaxg6W3SiToBYgESFRGk1wiCRGJYF9jqk1Uw"}, + "cats": "not cats", + "cheese": [ + {"/":"zb2rhisguzLFRJaxg6W3SiToBYgESFRGk1wiCRGJYF9jqk1Uw"}, + {"/":"zb2rhisguzLFRJaxg6W3SiToBYgESFRGk1wiCRGJYF9jqk1Uw"}, + {"/":"zb2rhisguzLFRJaxg6W3SiToBYgESFRGk1wiCRGJYF9jqk1Uw"}, + {"/":"zb2rhisguzLFRJaxg6W3SiToBYgESFRGk1wiCRGJYF9jqk1Uw"} + ] +}` + n, err := FromJson(bytes.NewReader([]byte(data)), mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + if err := assertCid(n.Cid(), "zdpuAqdmDwJ7oDv9cD4hp3yjXgWe9yhZDzRaFbRPin1c4Dz1y"); err != nil { + t.Fatal(err) + } + + c, ok := n.obj.(map[interface{}]interface{})["something"].(*cid.Cid) + if !ok { + t.Fatal("expected a cid") + } + + if c.String() != "zb2rhisguzLFRJaxg6W3SiToBYgESFRGk1wiCRGJYF9jqk1Uw" { + t.Fatal("cid unmarshaled wrong") + } +} + +func TestResolvedValIsJsonable(t *testing.T) { + data := `{ + "foo": { + "bar": 1, + "baz": 2 + } + }` + n, err := FromJson(strings.NewReader(data), mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + if err := assertCid(n.Cid(), "zdpuAku712jAPTQBrP58frxKxeAcVZZczXNCMwBPcKJJDZWdn"); err != nil { + t.Fatal(err) + } + + val, _, err := n.Resolve([]string{"foo"}) + if err != nil { + t.Fatal(err) + } + + out, err := json.Marshal(val) + if err != nil { + t.Fatal(err) + } + + if string(out) != `{"bar":1,"baz":2}` { + t.Fatal("failed to get expected json") + } +} + +func TestExamples(t *testing.T) { + examples := map[string]string{ + "[null]": "zdpuAzexuLRNr1owELqyN3ofh6yWVVKDq5wjFfmVDFbeXBHdj", + "[]": "zdpuAtQy7GSHNcZxdBfmtowdL1d2WAFjJBwb6WAEfFJ6T4Gbi", + "{}": "zdpuAyTBnYSugBZhqJuLsNpzjmAjSmxDqBbtAqXMtsvxiN2v3", + "null": "zdpuAxKCBsAKQpEw456S49oVDkWJ9PZa44KGRfVBWHiXN3UH8", + "1": "zdpuB2pwLskBDu5PZE2sepLyc3SRFPFgVXmnpzXVtWgam25kY", + "[1]": "zdpuB31oq9uvbqcSTySbWhD9NMBJDjsUXKtyQNhFAsYNbYH95", + "true": "zdpuAo6JPKbsmgmtujhh7mGywsAwPRmtyAYZBPKYYRjyLujD1", + `{"a":"IPFS"}`: "zdpuB3AZ71ccMjBB9atM97R4wSaCYjGyztnHnjUu93t4B2XqY", + `{"a":"IPFS","b":null,"c":[1]}`: "zdpuAyoYWNEe6xcGhkYk2SUfc7Rtbk4GkmZCrNAAnpft4Mmj5", + `{"a":[]}`: "zdpuAmMgJUCDGT4WhHAych8XpSVKQXEwsWhzQhhssr8542KXw", + } + for originalJson, expcid := range examples { + n, err := FromJson(bytes.NewReader([]byte(originalJson)), mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + if err := assertCid(n.Cid(), expcid); err != nil { + t.Fatalf("for object %s: %s", originalJson, err) + } + + cbor := n.RawData() + node, err := Decode(cbor, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + node, err = Decode(cbor, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + jsonBytes, err := node.MarshalJSON() + json := string(jsonBytes) + if json != originalJson { + t.Fatal("marshaled to incorrect JSON: " + json) + } + } +} + +func TestCanonicalize(t *testing.T) { + b, err := ioutil.ReadFile("test_objects/non-canon.cbor") + if err != nil { + t.Fatal(err) + } + nd1, err := Decode(b, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + if bytes.Equal(b, nd1.RawData()) { + t.Fatal("failed to canonicalize node") + } + + if err := assertCid(nd1.Cid(), "zdpuAmxF8q6iTUtkB3xtEYzmc5Sw762qwQJftt5iW8NTWLtjC"); err != nil { + t.Fatal(err) + } + + nd2, err := Decode(nd1.RawData(), mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + + if !nd2.Cid().Equals(nd1.Cid()) || !bytes.Equal(nd2.RawData(), nd1.RawData()) { + t.Fatal("re-decoding a canonical node should be idempotent") + } +} + +func TestStableCID(t *testing.T) { + b, err := ioutil.ReadFile("test_objects/non-canon.cbor") + if err != nil { + t.Fatal(err) + } + + hash, err := mh.Sum(b, mh.SHA2_256, -1) + if err != nil { + t.Fatal(err) + } + c := cid.NewCidV1(cid.DagCBOR, hash) + + badBlock, err := blocks.NewBlockWithCid(b, c) + if err != nil { + t.Fatal(err) + } + badNode, err := DecodeBlock(badBlock) + if err != nil { + t.Fatal(err) + } + + if !badBlock.Cid().Equals(badNode.Cid()) { + t.Fatal("CIDs not stable") + } +} diff --git a/tar/format.go b/tar/format.go index 38d681a..8e9d661 100644 --- a/tar/format.go +++ b/tar/format.go @@ -8,12 +8,12 @@ import ( "io" "strings" - importer "github.com/Casper-dev/Casper-server/importer" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - dag "github.com/Casper-dev/Casper-server/merkledag" - dagutil "github.com/Casper-dev/Casper-server/merkledag/utils" - path "github.com/Casper-dev/Casper-server/path" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + importer "gitlab.com/casperDev/Casper-server/importer" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + dag "gitlab.com/casperDev/Casper-server/merkledag" + dagutil "gitlab.com/casperDev/Casper-server/merkledag/utils" + path "gitlab.com/casperDev/Casper-server/path" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" diff --git a/test/bench/bench_cli_ipfs_add/main.go b/test/bench/bench_cli_ipfs_add/main.go index cfb697f..a1d7726 100644 --- a/test/bench/bench_cli_ipfs_add/main.go +++ b/test/bench/bench_cli_ipfs_add/main.go @@ -10,9 +10,9 @@ import ( "path" "testing" - "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" - "github.com/Casper-dev/Casper-server/repo/config" - "github.com/Casper-dev/Casper-server/thirdparty/unit" + "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" + "gitlab.com/casperDev/Casper-server/repo/config" + "gitlab.com/casperDev/Casper-server/thirdparty/unit" ) var ( diff --git a/test/bench/offline_add/main.go b/test/bench/offline_add/main.go index 9c5a057..80cfc75 100644 --- a/test/bench/offline_add/main.go +++ b/test/bench/offline_add/main.go @@ -9,9 +9,9 @@ import ( "path" "testing" - "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" - "github.com/Casper-dev/Casper-server/repo/config" - "github.com/Casper-dev/Casper-server/thirdparty/unit" + "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" + "gitlab.com/casperDev/Casper-server/repo/config" + "gitlab.com/casperDev/Casper-server/thirdparty/unit" ) func main() { diff --git a/test/integration/addcat_test.go b/test/integration/addcat_test.go index 2363030..5413f61 100644 --- a/test/integration/addcat_test.go +++ b/test/integration/addcat_test.go @@ -11,16 +11,18 @@ import ( "time" context "context" - random "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" - "github.com/Casper-dev/Casper-server/core" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" - mock "github.com/Casper-dev/Casper-server/core/mock" - "github.com/Casper-dev/Casper-server/thirdparty/unit" + random "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" + pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" mocknet "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/net/mock" + + "gitlab.com/casperDev/Casper-server/core" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + mock "gitlab.com/casperDev/Casper-server/core/mock" + "gitlab.com/casperDev/Casper-server/thirdparty/unit" ) var log = logging.Logger("epictest") diff --git a/test/integration/bench_cat_test.go b/test/integration/bench_cat_test.go index 927c1c5..3aac7db 100644 --- a/test/integration/bench_cat_test.go +++ b/test/integration/bench_cat_test.go @@ -8,13 +8,14 @@ import ( "testing" context "context" - "github.com/Casper-dev/Casper-server/core" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" - mock "github.com/Casper-dev/Casper-server/core/mock" - "github.com/Casper-dev/Casper-server/thirdparty/unit" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" mocknet "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/net/mock" + + "gitlab.com/casperDev/Casper-server/core" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + mock "gitlab.com/casperDev/Casper-server/core/mock" + "gitlab.com/casperDev/Casper-server/thirdparty/unit" ) func BenchmarkCat1MB(b *testing.B) { benchmarkVarCat(b, unit.MB*1) } diff --git a/test/integration/bench_test.go b/test/integration/bench_test.go index a1503b6..ac95e12 100644 --- a/test/integration/bench_test.go +++ b/test/integration/bench_test.go @@ -3,8 +3,9 @@ package integrationtest import ( "testing" - "github.com/Casper-dev/Casper-server/thirdparty/unit" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" + + "gitlab.com/casperDev/Casper-server/thirdparty/unit" ) func benchmarkAddCat(numBytes int64, conf testutil.LatencyConfig, b *testing.B) { diff --git a/test/integration/bitswap_wo_routing_test.go b/test/integration/bitswap_wo_routing_test.go index 1429a0c..9cd8a0f 100644 --- a/test/integration/bitswap_wo_routing_test.go +++ b/test/integration/bitswap_wo_routing_test.go @@ -4,10 +4,11 @@ import ( "bytes" "testing" - "github.com/Casper-dev/Casper-server/core" - "github.com/Casper-dev/Casper-server/core/mock" "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format" + "gitlab.com/casperDev/Casper-server/core" + "gitlab.com/casperDev/Casper-server/core/mock" + context "context" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" mocknet "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/net/mock" diff --git a/test/integration/three_legged_cat_test.go b/test/integration/three_legged_cat_test.go index a8cbe5e..e474794 100644 --- a/test/integration/three_legged_cat_test.go +++ b/test/integration/three_legged_cat_test.go @@ -10,13 +10,14 @@ import ( context "context" - core "github.com/Casper-dev/Casper-server/core" - coreunix "github.com/Casper-dev/Casper-server/core/coreunix" - mock "github.com/Casper-dev/Casper-server/core/mock" - "github.com/Casper-dev/Casper-server/thirdparty/unit" pstore "gx/ipfs/QmPgDWmTmuzvP7QE5zwo1TmjbJme9pmZHNujB2453jkCTr/go-libp2p-peerstore" testutil "gx/ipfs/QmWRCn8vruNAzHx8i6SAXinuheRitKEGu8c7m26stKvsYx/go-testutil" mocknet "gx/ipfs/QmefgzMbKZYsmHFkLqxgaTBG9ypeEjrdWRD5WXH4j1cWDL/go-libp2p/p2p/net/mock" + + core "gitlab.com/casperDev/Casper-server/core" + coreunix "gitlab.com/casperDev/Casper-server/core/coreunix" + mock "gitlab.com/casperDev/Casper-server/core/mock" + "gitlab.com/casperDev/Casper-server/thirdparty/unit" ) func TestThreeLeggedCatTransfer(t *testing.T) { diff --git a/test/sharness/lib/random-dep.go b/test/sharness/lib/random-dep.go index 3b0f79b..0c84fb0 100644 --- a/test/sharness/lib/random-dep.go +++ b/test/sharness/lib/random-dep.go @@ -4,6 +4,6 @@ package randomdep import ( - _ "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" - _ "github.com/Casper-dev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random-files" + _ "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random" + _ "gitlab.com/casperDev/Casper-server/Godeps/_workspace/src/github.com/jbenet/go-random-files" ) diff --git a/thirdparty/datastore2/delayed.go b/thirdparty/datastore2/delayed.go index 1fa082a..9c70c32 100644 --- a/thirdparty/datastore2/delayed.go +++ b/thirdparty/datastore2/delayed.go @@ -4,7 +4,7 @@ import ( ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore" dsq "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/query" - delay "github.com/Casper-dev/Casper-server/thirdparty/delay" + delay "gitlab.com/casperDev/Casper-server/thirdparty/delay" ) func WithDelay(ds ds.Datastore, delay delay.D) ds.Datastore { diff --git a/thirdparty/tar/extractor.go b/thirdparty/tar/extractor.go index b84926b..25d939c 100644 --- a/thirdparty/tar/extractor.go +++ b/thirdparty/tar/extractor.go @@ -8,9 +8,12 @@ import ( gopath "path" fp "path/filepath" "strings" + + "gitlab.com/casperDev/Casper-server/casper/crypto" ) type Extractor struct { + Password []byte Path string Progress func(int64) int64 } @@ -107,6 +110,10 @@ func (te *Extractor) extractFile(h *tar.Header, r *tar.Reader, depth int, rootEx } defer file.Close() + if te.Password != nil { + return copyWithProgress(file, crypto.NewAESReader(r, te.Password), te.Progress) + } + return copyWithProgress(file, r, te.Progress) } diff --git a/unixfs/archive/archive.go b/unixfs/archive/archive.go index 83b6058..78c13aa 100644 --- a/unixfs/archive/archive.go +++ b/unixfs/archive/archive.go @@ -7,9 +7,9 @@ import ( "io" "path" - mdag "github.com/Casper-dev/Casper-server/merkledag" - tar "github.com/Casper-dev/Casper-server/unixfs/archive/tar" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + tar "gitlab.com/casperDev/Casper-server/unixfs/archive/tar" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/unixfs/archive/tar/writer.go b/unixfs/archive/tar/writer.go index 1c5c95e..6199570 100644 --- a/unixfs/archive/tar/writer.go +++ b/unixfs/archive/tar/writer.go @@ -8,10 +8,10 @@ import ( "path" "time" - mdag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" - upb "github.com/Casper-dev/Casper-server/unixfs/pb" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + upb "gitlab.com/casperDev/Casper-server/unixfs/pb" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" diff --git a/unixfs/format.go b/unixfs/format.go index e0537bb..eb3eb1e 100644 --- a/unixfs/format.go +++ b/unixfs/format.go @@ -6,9 +6,10 @@ package unixfs import ( "errors" - dag "github.com/Casper-dev/Casper-server/merkledag" - pb "github.com/Casper-dev/Casper-server/unixfs/pb" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" + + dag "gitlab.com/casperDev/Casper-server/merkledag" + pb "gitlab.com/casperDev/Casper-server/unixfs/pb" ) const ( @@ -165,6 +166,10 @@ func (n *FSNode) RemoveBlockSize(i int) { n.blocksizes = append(n.blocksizes[:i], n.blocksizes[i+1:]...) } +func (n *FSNode) BlockSizes() []uint64 { + return n.blocksizes[:] +} + func (n *FSNode) GetBytes() ([]byte, error) { pbn := new(pb.Data) pbn.Type = &n.Type diff --git a/unixfs/format_test.go b/unixfs/format_test.go index c94b792..faf377a 100644 --- a/unixfs/format_test.go +++ b/unixfs/format_test.go @@ -6,7 +6,7 @@ import ( proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" - pb "github.com/Casper-dev/Casper-server/unixfs/pb" + pb "gitlab.com/casperDev/Casper-server/unixfs/pb" ) func TestFSNode(t *testing.T) { diff --git a/unixfs/hamt/hamt.go b/unixfs/hamt/hamt.go index c53d894..0babfdf 100644 --- a/unixfs/hamt/hamt.go +++ b/unixfs/hamt/hamt.go @@ -27,9 +27,9 @@ import ( "math/big" "os" - dag "github.com/Casper-dev/Casper-server/merkledag" - format "github.com/Casper-dev/Casper-server/unixfs" - upb "github.com/Casper-dev/Casper-server/unixfs/pb" + dag "gitlab.com/casperDev/Casper-server/merkledag" + format "gitlab.com/casperDev/Casper-server/unixfs" + upb "gitlab.com/casperDev/Casper-server/unixfs/pb" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/unixfs/hamt/hamt_stress_test.go b/unixfs/hamt/hamt_stress_test.go index a4a79d1..6f5f17a 100644 --- a/unixfs/hamt/hamt_stress_test.go +++ b/unixfs/hamt/hamt_stress_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - dag "github.com/Casper-dev/Casper-server/merkledag" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" - ft "github.com/Casper-dev/Casper-server/unixfs" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" + ft "gitlab.com/casperDev/Casper-server/unixfs" ) func getNames(prefix string, count int) []string { diff --git a/unixfs/hamt/hamt_test.go b/unixfs/hamt/hamt_test.go index c1bb53a..f4f084c 100644 --- a/unixfs/hamt/hamt_test.go +++ b/unixfs/hamt/hamt_test.go @@ -9,10 +9,10 @@ import ( "testing" "time" - dag "github.com/Casper-dev/Casper-server/merkledag" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" - dagutils "github.com/Casper-dev/Casper-server/merkledag/utils" - ft "github.com/Casper-dev/Casper-server/unixfs" + dag "gitlab.com/casperDev/Casper-server/merkledag" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" + dagutils "gitlab.com/casperDev/Casper-server/merkledag/utils" + ft "gitlab.com/casperDev/Casper-server/unixfs" ) func shuffle(seed int64, arr []string) { diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index eb1406f..8c84de1 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - mdag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" - ftpb "github.com/Casper-dev/Casper-server/unixfs/pb" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + ftpb "gitlab.com/casperDev/Casper-server/unixfs/pb" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" diff --git a/unixfs/io/dagreader_test.go b/unixfs/io/dagreader_test.go index 3ea177a..87d68a4 100644 --- a/unixfs/io/dagreader_test.go +++ b/unixfs/io/dagreader_test.go @@ -7,12 +7,12 @@ import ( "strings" "testing" - mdag "github.com/Casper-dev/Casper-server/merkledag" - "github.com/Casper-dev/Casper-server/unixfs" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + "gitlab.com/casperDev/Casper-server/unixfs" context "context" - testu "github.com/Casper-dev/Casper-server/unixfs/test" + testu "gitlab.com/casperDev/Casper-server/unixfs/test" ) func TestBasicRead(t *testing.T) { diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index 394da7b..105f2cd 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -5,11 +5,12 @@ import ( "fmt" "os" - mdag "github.com/Casper-dev/Casper-server/merkledag" - format "github.com/Casper-dev/Casper-server/unixfs" - hamt "github.com/Casper-dev/Casper-server/unixfs/hamt" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + format "gitlab.com/casperDev/Casper-server/unixfs" + hamt "gitlab.com/casperDev/Casper-server/unixfs/hamt" + node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/unixfs/io/dirbuilder_test.go b/unixfs/io/dirbuilder_test.go index 260b12b..d6fadb0 100644 --- a/unixfs/io/dirbuilder_test.go +++ b/unixfs/io/dirbuilder_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" - mdtest "github.com/Casper-dev/Casper-server/merkledag/test" - ft "github.com/Casper-dev/Casper-server/unixfs" + mdtest "gitlab.com/casperDev/Casper-server/merkledag/test" + ft "gitlab.com/casperDev/Casper-server/unixfs" ) func TestEmptyNode(t *testing.T) { diff --git a/unixfs/io/pbdagreader.go b/unixfs/io/pbdagreader.go index 92bc62e..d942585 100644 --- a/unixfs/io/pbdagreader.go +++ b/unixfs/io/pbdagreader.go @@ -6,9 +6,9 @@ import ( "fmt" "io" - mdag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" - ftpb "github.com/Casper-dev/Casper-server/unixfs/pb" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + ftpb "gitlab.com/casperDev/Casper-server/unixfs/pb" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" ) diff --git a/unixfs/io/resolve.go b/unixfs/io/resolve.go index 912a1d6..350fee1 100644 --- a/unixfs/io/resolve.go +++ b/unixfs/io/resolve.go @@ -3,9 +3,9 @@ package io import ( "context" - dag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" - hamt "github.com/Casper-dev/Casper-server/unixfs/hamt" + dag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + hamt "gitlab.com/casperDev/Casper-server/unixfs/hamt" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" ) diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go index ef18ff3..5b47b02 100644 --- a/unixfs/mod/dagmodifier.go +++ b/unixfs/mod/dagmodifier.go @@ -7,12 +7,12 @@ import ( "fmt" "io" - chunk "github.com/Casper-dev/Casper-server/importer/chunk" - help "github.com/Casper-dev/Casper-server/importer/helpers" - trickle "github.com/Casper-dev/Casper-server/importer/trickle" - mdag "github.com/Casper-dev/Casper-server/merkledag" - ft "github.com/Casper-dev/Casper-server/unixfs" - uio "github.com/Casper-dev/Casper-server/unixfs/io" + chunk "gitlab.com/casperDev/Casper-server/importer/chunk" + help "gitlab.com/casperDev/Casper-server/importer/helpers" + trickle "gitlab.com/casperDev/Casper-server/importer/trickle" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + ft "gitlab.com/casperDev/Casper-server/unixfs" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format" diff --git a/unixfs/mod/dagmodifier_test.go b/unixfs/mod/dagmodifier_test.go index 529f82e..e8f0dc1 100644 --- a/unixfs/mod/dagmodifier_test.go +++ b/unixfs/mod/dagmodifier_test.go @@ -7,11 +7,11 @@ import ( "io/ioutil" "testing" - h "github.com/Casper-dev/Casper-server/importer/helpers" - trickle "github.com/Casper-dev/Casper-server/importer/trickle" + h "gitlab.com/casperDev/Casper-server/importer/helpers" + trickle "gitlab.com/casperDev/Casper-server/importer/trickle" - uio "github.com/Casper-dev/Casper-server/unixfs/io" - testu "github.com/Casper-dev/Casper-server/unixfs/test" + uio "gitlab.com/casperDev/Casper-server/unixfs/io" + testu "gitlab.com/casperDev/Casper-server/unixfs/test" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" ) diff --git a/unixfs/test/utils.go b/unixfs/test/utils.go index b4f3754..b249c40 100644 --- a/unixfs/test/utils.go +++ b/unixfs/test/utils.go @@ -8,12 +8,12 @@ import ( "io/ioutil" "testing" - "github.com/Casper-dev/Casper-server/importer/chunk" - h "github.com/Casper-dev/Casper-server/importer/helpers" - trickle "github.com/Casper-dev/Casper-server/importer/trickle" - mdag "github.com/Casper-dev/Casper-server/merkledag" - mdagmock "github.com/Casper-dev/Casper-server/merkledag/test" - ft "github.com/Casper-dev/Casper-server/unixfs" + "gitlab.com/casperDev/Casper-server/importer/chunk" + h "gitlab.com/casperDev/Casper-server/importer/helpers" + trickle "gitlab.com/casperDev/Casper-server/importer/trickle" + mdag "gitlab.com/casperDev/Casper-server/merkledag" + mdagmock "gitlab.com/casperDev/Casper-server/merkledag/test" + ft "gitlab.com/casperDev/Casper-server/unixfs" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format"