Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement 'bacalhau get' for fetching job results without an IPFS binary. #273

Merged
merged 1 commit into from
Jun 23, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions cmd/bacalhau/get.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package bacalhau

import (
"context"
"fmt"
"path/filepath"

"github.com/filecoin-project/bacalhau/pkg/ipfs"
"github.com/filecoin-project/bacalhau/pkg/system"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
)

var getCmdFlags = struct {
ipfsURL string
outputDir string
}{
ipfsURL: "ipfs.io",
outputDir: ".",
}

func init() {
getCmd.Flags().StringVar(&getCmdFlags.ipfsURL, "ipfs-url",
getCmdFlags.ipfsURL, "URL of the IPFS gateway to use.")
getCmd.Flags().StringVar(&getCmdFlags.outputDir, "output-dir",
getCmdFlags.outputDir, "Directory to write the output to.")
}

var getCmd = &cobra.Command{
Use: "get",
Short: "Get the results of a job",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
cm := system.NewCleanupManager()
defer cm.Cleanup()

log.Debug().Msgf("Fetching results of job '%s'...", args[0])
job, ok, err := getAPIClient().Get(context.Background(), args[0])
if err != nil {
return err
}
if !ok {
return fmt.Errorf("job not found")
}

var resultCIDs []string
for _, jobState := range job.State {
if jobState.ResultsId != "" {
resultCIDs = append(resultCIDs, jobState.ResultsId)
}
}
log.Debug().Msgf("Job has result CIDs: %v", resultCIDs)

log.Debug().Msg("Spinning up IPFS client...")
cl, err := ipfs.NewClient(cm)
if err != nil {
return err
}

for _, cid := range resultCIDs {
outputDir := filepath.Join(getCmdFlags.outputDir, cid)
log.Debug().Msgf("Downloading result CID '%s' to '%s'...",
cid, outputDir)

err = cl.Get(context.Background(), cid, outputDir)
if err != nil {
return err
}
}

return nil
},
}
1 change: 1 addition & 0 deletions cmd/bacalhau/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ var apiPort int
func init() {
RootCmd.AddCommand(serveCmd)
RootCmd.AddCommand(runCmd)
RootCmd.AddCommand(getCmd)
RootCmd.AddCommand(listCmd)
RootCmd.AddCommand(devstackCmd)
RootCmd.PersistentFlags().StringVar(
Expand Down
161 changes: 116 additions & 45 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,25 @@ go 1.17
require (
github.com/docker/docker v20.10.14+incompatible
github.com/google/uuid v1.3.0
github.com/ipfs/go-ipfs v0.13.0-rc1
github.com/ipfs/go-ipfs-files v0.1.1
github.com/ipfs/go-ipfs-http-client v0.3.1
github.com/ipfs/interface-go-ipfs-core v0.7.0
github.com/jedib0t/go-pretty/v6 v6.2.7
github.com/libp2p/go-libp2p v0.18.0
github.com/libp2p/go-libp2p v0.19.4
github.com/libp2p/go-libp2p-core v0.15.1
github.com/libp2p/go-libp2p-pubsub v0.6.1
github.com/moby/moby v20.10.15+incompatible
github.com/multiformats/go-multiaddr v0.5.0
github.com/multiformats/go-multiaddr v0.6.0
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/rs/zerolog v1.26.1
github.com/spf13/cobra v1.2.1
github.com/spf13/viper v1.12.0
github.com/stretchr/testify v1.7.2
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0
go.opentelemetry.io/otel v1.7.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.7.0
go.opentelemetry.io/otel/sdk v1.7.0
go.opentelemetry.io/otel/trace v1.7.0
Expand All @@ -30,92 +32,142 @@ require (
)

require (
bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512 // indirect
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Stebalien/go-bitfield v0.0.1 // indirect
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd v0.22.0-beta // indirect
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/btcsuite/btcd v0.22.1 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/containerd/cgroups v1.0.3 // indirect
github.com/containerd/containerd v1.6.1 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/cskr/pubsub v1.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/dgraph-io/badger v1.6.2 // indirect
github.com/dgraph-io/ristretto v0.0.2 // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/elastic/gosigar v0.12.0 // indirect
github.com/felixge/httpsnoop v1.0.2 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/godbus/dbus/v5 v5.0.6 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/gopherjs/gopherjs v0.0.0-20220104163920-15ed2e8cf2bd // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huin/goupnp v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.0.0 // indirect
github.com/ipfs/go-bitswap v0.6.0 // indirect
github.com/ipfs/go-block-format v0.0.3 // indirect
github.com/ipfs/go-blockservice v0.3.0 // indirect
github.com/ipfs/go-cid v0.1.0 // indirect
github.com/ipfs/go-datastore v0.5.0 // indirect
github.com/ipfs/go-cidutil v0.0.2 // indirect
github.com/ipfs/go-datastore v0.5.1 // indirect
github.com/ipfs/go-ds-badger v0.3.0 // indirect
github.com/ipfs/go-ds-flatfs v0.5.1 // indirect
github.com/ipfs/go-ds-leveldb v0.5.0 // indirect
github.com/ipfs/go-ds-measure v0.2.0 // indirect
github.com/ipfs/go-fetcher v1.6.1 // indirect
github.com/ipfs/go-filestore v1.2.0 // indirect
github.com/ipfs/go-fs-lock v0.0.7 // indirect
github.com/ipfs/go-graphsync v0.13.1 // indirect
github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
github.com/ipfs/go-ipfs-cmds v0.6.0 // indirect
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
github.com/ipfs/go-ipfs-cmds v0.8.1 // indirect
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.1.0 // indirect
github.com/ipfs/go-ipfs-files v0.0.8 // indirect
github.com/ipfs/go-ipfs-exchange-offline v0.2.0 // indirect
github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
github.com/ipfs/go-ipfs-pinner v0.2.1 // indirect
github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
github.com/ipfs/go-ipfs-provider v0.7.1 // indirect
github.com/ipfs/go-ipfs-routing v0.2.1 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
github.com/ipfs/go-ipld-format v0.4.0 // indirect
github.com/ipfs/go-ipld-legacy v0.1.0 // indirect
github.com/ipfs/go-ipld-git v0.1.1 // indirect
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
github.com/ipfs/go-ipns v0.1.2 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.0 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-merkledag v0.6.0 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-path v0.1.1 // indirect
github.com/ipfs/go-unixfs v0.2.5 // indirect
github.com/ipfs/go-mfs v0.2.1 // indirect
github.com/ipfs/go-namesys v0.5.0 // indirect
github.com/ipfs/go-path v0.3.0 // indirect
github.com/ipfs/go-peertaskqueue v0.7.1 // indirect
github.com/ipfs/go-unixfs v0.3.1 // indirect
github.com/ipfs/go-unixfsnode v1.4.0 // indirect
github.com/ipfs/go-verifcid v0.0.1 // indirect
github.com/ipld/go-codec-dagpb v1.3.0 // indirect
github.com/ipld/go-ipld-prime v0.11.0 // indirect
github.com/ipld/go-codec-dagpb v1.4.0 // indirect
github.com/ipld/go-ipld-prime v0.16.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/klauspost/compress v1.15.6 // indirect
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
github.com/klauspost/cpuid/v2 v2.0.14 // indirect
github.com/koron/go-ssdp v0.0.2 // indirect
github.com/libp2p/go-buffer-pool v0.0.2 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-conn-security-multistream v0.3.0 // indirect
github.com/libp2p/go-doh-resolver v0.4.0 // indirect
github.com/libp2p/go-eventbus v0.2.1 // indirect
github.com/libp2p/go-flow-metrics v0.0.3 // indirect
github.com/libp2p/go-libp2p-asn-util v0.1.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
github.com/libp2p/go-libp2p-blankhost v0.3.0 // indirect
github.com/libp2p/go-libp2p-discovery v0.6.0 // indirect
github.com/libp2p/go-libp2p-mplex v0.6.0 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.15.0 // indirect
github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect
github.com/libp2p/go-libp2p-loggables v0.1.0 // indirect
github.com/libp2p/go-libp2p-mplex v0.7.0 // indirect
github.com/libp2p/go-libp2p-nat v0.1.0 // indirect
github.com/libp2p/go-libp2p-noise v0.3.0 // indirect
github.com/libp2p/go-libp2p-noise v0.4.0 // indirect
github.com/libp2p/go-libp2p-peerstore v0.6.0 // indirect
github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect
github.com/libp2p/go-libp2p-quic-transport v0.16.1 // indirect
github.com/libp2p/go-libp2p-resource-manager v0.1.5 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.5.0 // indirect
github.com/libp2p/go-libp2p-quic-transport v0.17.0 // indirect
github.com/libp2p/go-libp2p-record v0.1.3 // indirect
github.com/libp2p/go-libp2p-resource-manager v0.3.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.2.3 // indirect
github.com/libp2p/go-libp2p-swarm v0.10.2 // indirect
github.com/libp2p/go-libp2p-tls v0.3.1 // indirect
github.com/libp2p/go-libp2p-tls v0.4.1 // indirect
github.com/libp2p/go-libp2p-transport-upgrader v0.7.1 // indirect
github.com/libp2p/go-libp2p-yamux v0.8.2 // indirect
github.com/libp2p/go-mplex v0.6.0 // indirect
github.com/libp2p/go-msgio v0.1.0 // indirect
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
github.com/libp2p/go-libp2p-yamux v0.9.1 // indirect
github.com/libp2p/go-mplex v0.7.0 // indirect
github.com/libp2p/go-msgio v0.2.0 // indirect
github.com/libp2p/go-nat v0.1.0 // indirect
github.com/libp2p/go-netroute v0.2.0 // indirect
github.com/libp2p/go-openssl v0.0.7 // indirect
Expand All @@ -124,20 +176,20 @@ require (
github.com/libp2p/go-stream-muxer-multistream v0.4.0 // indirect
github.com/libp2p/go-tcp-transport v0.5.1 // indirect
github.com/libp2p/go-ws-transport v0.6.0 // indirect
github.com/libp2p/go-yamux/v3 v3.0.2 // indirect
github.com/lucas-clemente/quic-go v0.25.0 // indirect
github.com/libp2p/go-yamux/v3 v3.1.2 // indirect
github.com/libp2p/zeroconf/v2 v2.1.1 // indirect
github.com/lucas-clemente/quic-go v0.27.1 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.49 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
Expand All @@ -146,17 +198,18 @@ require (
github.com/multiformats/go-base36 v0.1.0 // indirect
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.0.3 // indirect
github.com/multiformats/go-multicodec v0.4.1 // indirect
github.com/multiformats/go-multihash v0.1.0 // indirect
github.com/multiformats/go-multistream v0.2.2 // indirect
github.com/multiformats/go-multibase v0.1.1 // indirect
github.com/multiformats/go-multicodec v0.5.0 // indirect
github.com/multiformats/go-multihash v0.2.0 // indirect
github.com/multiformats/go-multistream v0.3.0 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/openzipkin/zipkin-go v0.4.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
Expand All @@ -181,27 +234,45 @@ require (
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.3.0 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/tidwall/gjson v1.14.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/wI2L/jsondiff v0.2.0 // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 // indirect
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.7.0 // indirect
go.opentelemetry.io/otel/metric v0.30.0 // indirect
go.opentelemetry.io/proto/otlp v0.11.0 // indirect
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/dig v1.14.0 // indirect
go.uber.org/fx v1.16.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d // indirect
golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
golang.org/x/tools v0.1.11 // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/ini.v1 v1.66.6 // indirect
gopkg.in/square/go-jose.v2 v2.5.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
Loading