Merge pull request #61 with GRAPH_BACKEND

.gitignore

@@ -3,6 +3,7 @@
 *.coverprofile
 *.test
 *.orig
+*.exe
 *~
 
 .tarball

blockstore_caboose.go

@@ -41,7 +41,7 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS)
 	}
 
 	saturnOrchestratorClient := &http.Client{
-		Timeout: caboose.DefaultSaturnRequestTimeout,
+		Timeout: caboose.DefaultSaturnOrchestratorRequestTimeout,
 		Transport: &customTransport{
 			RoundTripper: &http.Transport{
 				DialContext: cdns.dialWithCachedDNS,
@@ -50,7 +50,7 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS)
 	}
 
 	saturnLoggerClient := &http.Client{
-		Timeout: caboose.DefaultSaturnRequestTimeout,
+		Timeout: caboose.DefaultSaturnLoggerRequestTimeout,
 		Transport: &customTransport{
 			AuthorizationBearerToken: os.Getenv(EnvSaturnLoggerSecret),
 			RoundTripper: &http.Transport{
@@ -60,7 +60,7 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS)
 	}
 
 	saturnRetrievalClient := &http.Client{
-		Timeout: caboose.DefaultSaturnRequestTimeout,
+		Timeout: caboose.DefaultSaturnCarRequestTimeout,
 		Transport: &customTransport{
 			RoundTripper: &http.Transport{
 				// Increasing concurrency defaults from http.DefaultTransport
@@ -93,10 +93,10 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS)
 
 		LoggingEndpoint: *loggURL,
 		LoggingClient:   saturnLoggerClient,
-		LoggingInterval: 5 * time.Second,
+		LoggingInterval: caboose.DefaultLoggingInterval,
 
 		DoValidation: true,
-		PoolRefresh:  5 * time.Minute,
+		PoolRefresh:  caboose.DefaultPoolRefreshInterval,
 		SaturnClient: saturnRetrievalClient,
 	})
 }

blockstore_proxy.go

@@ -10,8 +10,9 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/ipfs/bifrost-gateway/lib"
 	blockstore "github.com/ipfs/boxo/blockstore"
-	"github.com/ipfs/go-block-format"
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 )
 
@@ -25,7 +26,7 @@ const (
 	EnvProxyGateway = "PROXY_GATEWAY_URL"
 
 	DefaultProxyGateway = "http://127.0.0.1:8080"
-	DefaultKuboPRC      = "http://127.0.0.1:5001"
+	DefaultKuboRPC      = "http://127.0.0.1:5001"
 )
 
 type proxyBlockStore struct {
@@ -35,6 +36,37 @@ type proxyBlockStore struct {
 	rand       *rand.Rand
 }
 
+func (ps *proxyBlockStore) Fetch(ctx context.Context, path string, cb lib.DataCallback) error {
+	u, err := url.Parse(fmt.Sprintf("%s%s", ps.getRandomGatewayURL(), path))
+	if err != nil {
+		return err
+	}
+	goLog.Debugw("car fetch", "url", u)
+	resp, err := ps.httpClient.Do(&http.Request{
+		Method: http.MethodGet,
+		URL:    u,
+		Header: http.Header{
+			"Accept": []string{"application/vnd.ipld.car"},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("http error from car gateway: %s", resp.Status)
+	}
+
+	err = cb(path, resp.Body)
+	if err != nil {
+		resp.Body.Close()
+		return err
+	}
+	return resp.Body.Close()
+}
+
+var _ lib.CarFetcher = (*proxyBlockStore)(nil)
+
 func newProxyBlockStore(gatewayURL []string, cdns *cachedDNS) blockstore.Blockstore {
 	s := rand.NewSource(time.Now().Unix())
 	rand := rand.New(s)
@@ -71,6 +103,7 @@ func (ps *proxyBlockStore) fetch(ctx context.Context, c cid.Cid) (blocks.Block,
 	if err != nil {
 		return nil, err
 	}
+	goLog.Debugw("raw fetch", "url", u)
 	resp, err := ps.httpClient.Do(&http.Request{
 		Method: http.MethodGet,
 		URL:    u,

docs/environment-variables.md

@@ -5,6 +5,7 @@
 - [Configuration](#configuration)
   - [`KUBO_RPC_URL`](#kubo_rpc_url)
   - [`BLOCK_CACHE_SIZE`](#block_cache_size)
+  - [`GRAPH_BACKEND`](#graph_backend)
 - [Proxy Backend](#proxy-backend)
   - [`PROXY_GATEWAY_URL`](#proxy_gateway_url)
 - [Saturn Backend](#saturn-backend)
@@ -25,15 +26,33 @@
 
 ### `KUBO_RPC_URL`
 
+Default: see `DefaultKuboRPC`
+
 Single URL or a comma separated list of RPC endpoints that provide `/api/v0` from Kubo.
 
 We use this as temporary solution for IPNS Record routing until [IPIP-351](https://github.com/ipfs/specs/pull/351) ships with Kubo 0.19,
 and we also redirect some legacy `/api/v0` commands that need to be handled on `ipfs.io`.
 
 ### `BLOCK_CACHE_SIZE`
 
+Default: see `DefaultCacheBlockStoreSize`
+
 The size of in-memory [2Q cache](https://pkg.go.dev/github.com/hashicorp/golang-lru/v2#TwoQueueCache) with recently used and most requently used blocks.
 
+### `GRAPH_BACKEND`
+
+Default: `false`
+
+When set to `true`, requests to backend will use
+`?format=car&depth=..&bytes=..` in addition to `?format=raw` to reduce the
+number of round trips.
+
+This is an experimental feature that depends on `&depth=..&bytes=..`
+parameters.  Currently only `https://l1s.strn.pl` supports it, but our
+intention is to standardize it and add it to the
+[trustless gateway spec](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
+in the near feature.
+
 ## Proxy Backend
 
 ### `PROXY_GATEWAY_URL`

go.mod

@@ -3,110 +3,151 @@ module github.com/ipfs/bifrost-gateway
 go 1.19
 
 require (
-	github.com/filecoin-saturn/caboose v0.0.0-20230329141039-b4cc70873ccd
+	github.com/cskr/pubsub v1.0.2
+	github.com/filecoin-saturn/caboose v0.0.0-20230329203940-7c08345c244d
 	github.com/gogo/protobuf v1.3.2
 	github.com/hashicorp/golang-lru/v2 v2.0.1
-	github.com/ipfs/boxo v0.8.0-rc2.0.20230329082438-360b031ed895
+	github.com/ipfs/boxo v0.8.0-rc4
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.0
 	github.com/ipfs/go-ipld-format v0.4.0
 	github.com/ipfs/go-log/v2 v2.5.1
+	github.com/ipfs/go-unixfsnode v1.6.0
+	github.com/ipld/go-codec-dagpb v1.6.0
+	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/libp2p/go-libp2p v0.26.3
+	github.com/libp2p/go-libp2p-kad-dht v0.21.1
+	github.com/libp2p/go-libp2p-routing-helpers v0.4.0
+	github.com/multiformats/go-multicodec v0.8.1
+	github.com/multiformats/go-multihash v0.2.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.2
 	go.uber.org/atomic v1.10.0
+	go.uber.org/multierr v1.9.0
 	go.uber.org/zap v1.24.0
 )
 
 require (
 	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
 	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
 	github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
 	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
 	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
 	github.com/ipfs/go-ipns v0.3.0 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
-	github.com/ipfs/go-unixfsnode v1.6.0 // indirect
-	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
-	github.com/ipld/go-ipld-prime v0.20.0 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
+	github.com/koron/go-ssdp v0.0.3 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
 	github.com/libp2p/go-doh-resolver v0.4.0 // indirect
+	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-kad-dht v0.21.1 // indirect
 	github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect
 	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-routing-helpers v0.4.0 // indirect
 	github.com/libp2p/go-msgio v0.3.0 // indirect
+	github.com/libp2p/go-nat v0.1.0 // indirect
 	github.com/libp2p/go-netroute v0.2.1 // indirect
+	github.com/libp2p/go-reuseport v0.2.0 // indirect
+	github.com/libp2p/go-yamux/v4 v4.0.0 // indirect
+	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/miekg/dns v1.1.50 // indirect
+	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr v0.8.0 // indirect
 	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.1.1 // indirect
-	github.com/multiformats/go-multicodec v0.8.1 // indirect
-	github.com/multiformats/go-multihash v0.2.1 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/onsi/ginkgo/v2 v2.5.1 // indirect
 	github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.39.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/qtls-go1-19 v0.2.1 // indirect
+	github.com/quic-go/qtls-go1-20 v0.1.1 // indirect
+	github.com/quic-go/quic-go v0.33.0 // indirect
+	github.com/quic-go/webtransport-go v0.5.2 // indirect
+	github.com/raulk/go-watchdog v1.3.0 // indirect
 	github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
+	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel v1.14.0 // indirect
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
+	go.uber.org/dig v1.15.0 // indirect
+	go.uber.org/fx v1.18.2 // indirect
 	golang.org/x/crypto v0.6.0 // indirect
+	golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect
 	golang.org/x/mod v0.7.0 // indirect
 	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/tools v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
+	nhooyr.io/websocket v1.8.7 // indirect
 )