From 7610773f1ab4af29b1a05f869a7b17c96a333414 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 4 Sep 2023 15:46:05 +0200 Subject: [PATCH] refactor!: namesys package --- CHANGELOG.md | 15 ++ coreiface/options/name.go | 6 +- coreiface/options/namesys/opts.go | 131 ----------- gateway/blocks_backend.go | 46 ++-- gateway/gateway.go | 5 +- gateway/gateway_test.go | 64 ++++-- gateway/handler.go | 35 +-- gateway/handler_block.go | 2 +- gateway/handler_car.go | 2 +- gateway/handler_codec.go | 2 +- gateway/handler_defaults.go | 12 +- gateway/handler_tar.go | 2 +- gateway/handler_unixfs__redirects.go | 3 +- gateway/handler_unixfs_dir.go | 36 +-- gateway/handler_unixfs_dir_test.go | 2 +- gateway/handler_unixfs_file.go | 8 +- gateway/hostname_test.go | 4 +- gateway/metrics.go | 6 +- gateway/utilities_test.go | 58 +++-- namesys/base.go | 126 ----------- namesys/cache.go | 62 ------ namesys/{dns.go => dns_resolver.go} | 103 ++++----- namesys/dns_resolver_test.go | 197 ++++++++++++++++ namesys/dns_test.go | 184 --------------- namesys/interface.go | 235 +++++++++++++++----- namesys/{publisher.go => ipns_publisher.go} | 135 ++++++----- namesys/ipns_publisher_test.go | 121 ++++++++++ namesys/ipns_resolver.go | 158 +++++++++++++ namesys/ipns_resolver_test.go | 131 +++++++++++ namesys/namesys.go | 231 +++++++++---------- namesys/namesys_cache.go | 78 +++++++ namesys/namesys_test.go | 184 +++++++-------- namesys/publisher_test.go | 159 ------------- namesys/republisher/repub.go | 50 +++-- namesys/republisher/repub_test.go | 97 +++----- namesys/resolve/resolve.go | 56 ----- namesys/resolve_test.go | 132 ----------- namesys/routing.go | 147 ------------ namesys/tracing.go | 13 -- namesys/utilities.go | 146 ++++++++++++ 40 files changed, 1570 insertions(+), 1614 deletions(-) delete mode 100644 coreiface/options/namesys/opts.go delete mode 100644 namesys/base.go delete mode 100644 namesys/cache.go rename namesys/{dns.go => dns_resolver.go} (58%) create mode 100644 namesys/dns_resolver_test.go delete mode 100644 namesys/dns_test.go rename namesys/{publisher.go => ipns_publisher.go} (51%) create mode 100644 namesys/ipns_publisher_test.go create mode 100644 namesys/ipns_resolver.go create mode 100644 namesys/ipns_resolver_test.go create mode 100644 namesys/namesys_cache.go delete mode 100644 namesys/publisher_test.go delete mode 100644 namesys/resolve/resolve.go delete mode 100644 namesys/resolve_test.go delete mode 100644 namesys/routing.go delete mode 100644 namesys/tracing.go create mode 100644 namesys/utilities.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 111044407..3a22c73e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The following emojis are used to highlight certain changes: ### Added +* The gateway now sets a `Cache-Control` header for requests under the `/ipns/` namespace + if the TTL for the corresponding IPNS Records or DNSLink entities is known. + ### Changed * `boxo/gateway` @@ -35,6 +38,18 @@ The following emojis are used to highlight certain changes: - Eliminate `..` elements that begin a rooted path: that is, replace "`/..`" by "`/`" at the beginning of a path. * 🛠 The signature of `CoreAPI.ResolvePath` in `coreiface` has changed to now return the remainder segments as a second return value, matching the signature of `resolver.ResolveToLastNode`. +* 🛠 The `namesys` package has been refactored. The following are the largest modifications: + * The options in `coreiface/options/namesys` have been moved to `namesys` and their names + have been made more consistent. + * Many of the exported structs and functions have been renamed in order to be consistent with + the remaining packages. + * `namesys.Resolver.Resolve` now returns a TTL, in addition to the resolved path. If the + TTL is unknown, 0 is returned. `IPNSResolver` is able to resolve a TTL, while `DNSResolver` + is not. + * `namesys/resolver.ResolveIPNS` has been moved to `namesys.ResolveIPNS` and now returns a TTL + in addition to the resolved path. +* 🛠 The `gateway`'s `IPFSBackend.ResolveMutable` is now expected to return a TTL in addition to + the resolved path. If the TTL is unknown, 0 should be returned. ### Removed diff --git a/coreiface/options/name.go b/coreiface/options/name.go index 35e78c394..7b4b6a8fd 100644 --- a/coreiface/options/name.go +++ b/coreiface/options/name.go @@ -3,7 +3,7 @@ package options import ( "time" - ropts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/namesys" ) const ( @@ -21,7 +21,7 @@ type NamePublishSettings struct { type NameResolveSettings struct { Cache bool - ResolveOpts []ropts.ResolveOpt + ResolveOpts []namesys.ResolveOption } type ( @@ -123,7 +123,7 @@ func (nameOpts) Cache(cache bool) NameResolveOption { } } -func (nameOpts) ResolveOption(opt ropts.ResolveOpt) NameResolveOption { +func (nameOpts) ResolveOption(opt namesys.ResolveOption) NameResolveOption { return func(settings *NameResolveSettings) error { settings.ResolveOpts = append(settings.ResolveOpts, opt) return nil diff --git a/coreiface/options/namesys/opts.go b/coreiface/options/namesys/opts.go deleted file mode 100644 index ed568200b..000000000 --- a/coreiface/options/namesys/opts.go +++ /dev/null @@ -1,131 +0,0 @@ -package nsopts - -import ( - "time" -) - -const ( - // DefaultDepthLimit is the default depth limit used by Resolve. - DefaultDepthLimit = 32 - - // UnlimitedDepth allows infinite recursion in Resolve. You - // probably don't want to use this, but it's here if you absolutely - // trust resolution to eventually complete and can't put an upper - // limit on how many steps it will take. - UnlimitedDepth = 0 - - // DefaultIPNSRecordTTL specifies the time that the record can be cached - // before checking if its validity again. - DefaultIPNSRecordTTL = time.Minute - - // DefaultIPNSRecordEOL specifies the time that the network will cache IPNS - // records after being published. Records should be re-published before this - // interval expires. We use the same default expiration as the DHT. - DefaultIPNSRecordEOL = 48 * time.Hour -) - -// ResolveOpts specifies options for resolving an IPNS path -type ResolveOpts struct { - // Recursion depth limit - Depth uint - // The number of IPNS records to retrieve from the DHT - // (the best record is selected from this set) - DhtRecordCount uint - // The amount of time to wait for DHT records to be fetched - // and verified. A zero value indicates that there is no explicit - // timeout (although there is an implicit timeout due to dial - // timeouts within the DHT) - DhtTimeout time.Duration -} - -// DefaultResolveOpts returns the default options for resolving -// an IPNS path -func DefaultResolveOpts() ResolveOpts { - return ResolveOpts{ - Depth: DefaultDepthLimit, - DhtRecordCount: 16, - DhtTimeout: time.Minute, - } -} - -// ResolveOpt is used to set an option -type ResolveOpt func(*ResolveOpts) - -// Depth is the recursion depth limit -func Depth(depth uint) ResolveOpt { - return func(o *ResolveOpts) { - o.Depth = depth - } -} - -// DhtRecordCount is the number of IPNS records to retrieve from the DHT -func DhtRecordCount(count uint) ResolveOpt { - return func(o *ResolveOpts) { - o.DhtRecordCount = count - } -} - -// DhtTimeout is the amount of time to wait for DHT records to be fetched -// and verified. A zero value indicates that there is no explicit timeout -func DhtTimeout(timeout time.Duration) ResolveOpt { - return func(o *ResolveOpts) { - o.DhtTimeout = timeout - } -} - -// ProcessOpts converts an array of ResolveOpt into a ResolveOpts object -func ProcessOpts(opts []ResolveOpt) ResolveOpts { - rsopts := DefaultResolveOpts() - for _, option := range opts { - option(&rsopts) - } - return rsopts -} - -// PublishOptions specifies options for publishing an IPNS record. -type PublishOptions struct { - EOL time.Time - TTL time.Duration - CompatibleWithV1 bool -} - -// DefaultPublishOptions returns the default options for publishing an IPNS record. -func DefaultPublishOptions() PublishOptions { - return PublishOptions{ - EOL: time.Now().Add(DefaultIPNSRecordEOL), - TTL: DefaultIPNSRecordTTL, - } -} - -// PublishOption is used to set an option for PublishOpts. -type PublishOption func(*PublishOptions) - -// PublishWithEOL sets an EOL. -func PublishWithEOL(eol time.Time) PublishOption { - return func(o *PublishOptions) { - o.EOL = eol - } -} - -// PublishWithEOL sets a TTL. -func PublishWithTTL(ttl time.Duration) PublishOption { - return func(o *PublishOptions) { - o.TTL = ttl - } -} - -// PublishCompatibleWithV1 sets compatibility with IPNS Records V1. -func PublishCompatibleWithV1(compatible bool) PublishOption { - return func(o *PublishOptions) { - o.CompatibleWithV1 = compatible - } -} - -// ProcessPublishOptions converts an array of PublishOpt into a PublishOpts object. -func ProcessPublishOptions(opts []PublishOption) PublishOptions { - rsopts := DefaultPublishOptions() - for _, option := range opts { - option(&rsopts) - } - return rsopts -} diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index db5f68924..710771f24 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -8,18 +8,18 @@ import ( "io" "net/http" "strings" + "time" "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" - nsopts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/fetcher" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/ipld/merkledag" ufile "github.com/ipfs/boxo/ipld/unixfs/file" uio "github.com/ipfs/boxo/ipld/unixfs/io" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/namesys" - "github.com/ipfs/boxo/namesys/resolve" "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" blocks "github.com/ipfs/go-block-format" @@ -39,7 +39,6 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" mc "github.com/multiformats/go-multicodec" @@ -615,18 +614,23 @@ func (bb *BlocksBackend) getPathRoots(ctx context.Context, contentPath path.Immu return pathRoots, lastPath, remainder, nil } -func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { switch p.Namespace() { case path.IPNSNamespace: - p, err := resolve.ResolveIPNS(ctx, bb.namesys, p) + res, err := namesys.Resolve(ctx, bb.namesys, p) if err != nil { - return path.ImmutablePath{}, err + return path.ImmutablePath{}, 0, time.Time{}, err } - return path.NewImmutablePath(p) + ip, err := path.NewImmutablePath(res.Path) + if err != nil { + return path.ImmutablePath{}, 0, time.Time{}, err + } + return ip, res.TTL, res.LastMod, nil case path.IPFSNamespace: - return path.NewImmutablePath(p) + ip, err := path.NewImmutablePath(p) + return ip, 0, time.Time{}, err default: - return path.ImmutablePath{}, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + return path.ImmutablePath{}, 0, time.Time{}, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) } } @@ -635,28 +639,25 @@ func (bb *BlocksBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, return nil, NewErrorStatusCode(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) } - // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful - // round trips to the remote routing provider. - if mc.Code(c.Type()) != mc.Libp2pKey { - return nil, NewErrorStatusCode(errors.New("cid codec must be libp2p-key"), http.StatusBadRequest) - } - - // The value store expects the key itself to be encoded as a multihash. - id, err := peer.FromCid(c) + name, err := ipns.NameFromCid(c) if err != nil { - return nil, err + return nil, NewErrorStatusCode(err, http.StatusBadRequest) } - return bb.routing.GetValue(ctx, "/ipns/"+string(id)) + return bb.routing.GetValue(ctx, string(name.RoutingKey())) } func (bb *BlocksBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { if bb.namesys != nil { - p, err := bb.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + p, err := path.NewPath("/ipns/" + hostname) + if err != nil { + return nil, err + } + res, err := bb.namesys.Resolve(ctx, p, namesys.ResolveWithDepth(1)) if err == namesys.ErrResolveRecursion { err = nil } - return p, err + return res.Path, err } return nil, NewErrorStatusCode(errors.New("not implemented"), http.StatusNotImplemented) @@ -688,10 +689,11 @@ func (bb *BlocksBackend) ResolvePath(ctx context.Context, path path.ImmutablePat func (bb *BlocksBackend) resolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) { var err error if p.Namespace() == path.IPNSNamespace { - p, err = resolve.ResolveIPNS(ctx, bb.namesys, p) + res, err := namesys.Resolve(ctx, bb.namesys, p) if err != nil { return path.ImmutablePath{}, nil, err } + p = res.Path } if p.Namespace() != path.IPFSNamespace { diff --git a/gateway/gateway.go b/gateway/gateway.go index bfafa48e1..06d236d1f 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway/assets" @@ -369,11 +370,11 @@ type IPFSBackend interface { GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) // ResolveMutable takes a mutable path and resolves it into an immutable one. This means recursively resolving any - // DNSLink or IPNS records. + // DNSLink or IPNS records. It should also return a TTL. If the TTL is unknown, 0 should be returned. // // For example, given a mapping from `/ipns/dnslink.tld -> /ipns/ipns-id/mydirectory` and `/ipns/ipns-id` to // `/ipfs/some-cid`, the result of passing `/ipns/dnslink.tld/myfile` would be `/ipfs/some-cid/mydirectory/myfile`. - ResolveMutable(context.Context, path.Path) (path.ImmutablePath, error) + ResolveMutable(context.Context, path.Path) (path.ImmutablePath, time.Duration, time.Time, error) // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN. // Unlike ResolvePath, it does not perform recursive resolution. It only diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index d3a4ac9b5..e9cb1c150 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -37,11 +37,11 @@ func TestGatewayGet(t *testing.T) { return p } - backend.namesys["/ipns/example.com"] = path.FromCid(k.RootCid()) - backend.namesys["/ipns/working.example.com"] = k - backend.namesys["/ipns/double.example.com"] = mustMakeDNSLinkPath("working.example.com") - backend.namesys["/ipns/triple.example.com"] = mustMakeDNSLinkPath("double.example.com") - backend.namesys["/ipns/broken.example.com"] = mustMakeDNSLinkPath(k.RootCid().String()) + backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.FromCid(k.RootCid()), 0) + backend.namesys["/ipns/working.example.com"] = newMockNamesysItem(k, 0) + backend.namesys["/ipns/double.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath("working.example.com"), 0) + backend.namesys["/ipns/triple.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath("double.example.com"), 0) + backend.namesys["/ipns/broken.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath(k.RootCid().String()), 0) // We picked .man because: // 1. It's a valid TLD. // 2. Go treats it as the file extension for "man" files (even though @@ -49,7 +49,7 @@ func TestGatewayGet(t *testing.T) { // // Unfortunately, this may not work on all platforms as file type // detection is platform dependent. - backend.namesys["/ipns/example.man"] = k + backend.namesys["/ipns/example.man"] = newMockNamesysItem(k, 0) for _, test := range []struct { host string @@ -98,7 +98,7 @@ func TestPretty404(t *testing.T) { t.Logf("test server url: %s", ts.URL) host := "example.net" - backend.namesys["/ipns/"+host] = path.FromCid(root) + backend.namesys["/ipns/"+host] = newMockNamesysItem(path.FromCid(root), 0) for _, test := range []struct { path string @@ -158,7 +158,41 @@ func TestHeaders(t *testing.T) { dagCborRoots = dirRoots + "," + dagCborCID ) - t.Run("Cache-Control is not immutable on generated /ipfs/ HTML dir listings", func(t *testing.T) { + t.Run("Cache-Control uses TTL for /ipns/ when it is known", func(t *testing.T) { + t.Parallel() + + ts, backend, root := newTestServerAndNode(t, nil, "ipns-hostname-redirects.car") + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.FromCid(root), time.Second*30) + backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.FromCid(root), time.Second*55) + backend.namesys["/ipns/unknown.com"] = newMockNamesysItem(path.FromCid(root), 0) + + testCases := []struct { + path string + cacheControl string + }{ + {"/ipns/example.net/", "public, max-age=30"}, // As generated directory listing + {"/ipns/example.com/", "public, max-age=55"}, // As generated directory listing (different) + {"/ipns/unknown.com/", ""}, // As generated directory listing (unknown) + {"/ipns/example.net/foo/", "public, max-age=30"}, // As index.html directory listing + {"/ipns/example.net/foo/index.html", "public, max-age=30"}, // As deserialized UnixFS file + {"/ipns/example.net/?format=raw", "public, max-age=30"}, // As Raw block + {"/ipns/example.net/?format=dag-json", "public, max-age=30"}, // As DAG-JSON block + {"/ipns/example.net/?format=dag-cbor", "public, max-age=30"}, // As DAG-CBOR block + {"/ipns/example.net/?format=car", "public, max-age=30"}, // As CAR block + } + + for _, testCase := range testCases { + req := mustNewRequest(t, http.MethodGet, ts.URL+testCase.path, nil) + res := mustDoWithoutRedirect(t, req) + if testCase.cacheControl == "" { + assert.Empty(t, res.Header["Cache-Control"]) + } else { + assert.Equal(t, testCase.cacheControl, res.Header.Get("Cache-Control")) + } + } + }) + + t.Run("Cache-Control is not immutable on generated /ipfs/ HTML dir listings", func(t *testing.T) { req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+rootCID+"/", nil) res := mustDoWithoutRedirect(t, req) @@ -500,7 +534,7 @@ func TestRedirects(t *testing.T) { t.Parallel() ts, backend, root := newTestServerAndNode(t, nil, "ipns-hostname-redirects.car") - backend.namesys["/ipns/example.net"] = path.FromCid(root) + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.FromCid(root), 0) // make request to directory containing index.html req := mustNewRequest(t, http.MethodGet, ts.URL+"/foo", nil) @@ -535,7 +569,7 @@ func TestRedirects(t *testing.T) { t.Parallel() backend, root := newMockBackend(t, "redirects-spa.car") - backend.namesys["/ipns/example.com"] = path.FromCid(root) + backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ Headers: map[string][]string{}, @@ -672,8 +706,8 @@ func TestDeserializedResponses(t *testing.T) { t.Parallel() backend, root := newMockBackend(t, "fixtures.car") - backend.namesys["/ipns/trustless.com"] = path.FromCid(root) - backend.namesys["/ipns/trusted.com"] = path.FromCid(root) + backend.namesys["/ipns/trustless.com"] = newMockNamesysItem(path.FromCid(root), 0) + backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ Headers: map[string][]string{}, @@ -735,8 +769,8 @@ func (mb *errorMockBackend) GetCAR(ctx context.Context, path path.ImmutablePath, return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { - return path.ImmutablePath{}, mb.err +func (mb *errorMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { + return path.ImmutablePath{}, 0, time.Time{}, mb.err } func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { @@ -819,7 +853,7 @@ func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath path.Immut panic("i am panicking") } -func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { panic("i am panicking") } diff --git a/gateway/handler.go b/gateway/handler.go index 7e3ccbf48..44218975f 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -34,7 +34,7 @@ var log = logging.Logger("boxo/gateway") const ( ipfsPathPrefix = "/ipfs/" - ipnsPathPrefix = "/ipns/" + ipnsPathPrefix = ipns.NamespacePrefix immutableCacheControl = "public, max-age=29030400, immutable" ) @@ -194,6 +194,8 @@ type requestData struct { // Defined for non IPNS Record requests. immutablePath path.ImmutablePath + ttl time.Duration + lastMod time.Time // Defined if resolution has already happened. pathMetadata *ContentPathMetadata @@ -285,7 +287,7 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } if contentPath.Mutable() { - rq.immutablePath, err = i.backend.ResolveMutable(r.Context(), contentPath) + rq.immutablePath, rq.ttl, rq.lastMod, err = i.backend.ResolveMutable(r.Context(), contentPath) if err != nil { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) @@ -415,7 +417,7 @@ func panicHandler(w http.ResponseWriter) { } } -func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath path.Path, cid cid.Cid, responseFormat string) (modtime time.Time) { +func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath path.Path, ttl time.Duration, lastMod time.Time, cid cid.Cid, responseFormat string) (modtime time.Time) { // Best effort attempt to set an Etag based on the CID and response format. // Setting an ETag is handled separately for CARs and IPNS records. if etag := getEtag(r, cid, responseFormat); etag != "" { @@ -424,23 +426,24 @@ func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath // Set Cache-Control and Last-Modified based on contentPath properties if contentPath.Mutable() { - // mutable namespaces such as /ipns/ can't be cached forever - - // For now we set Last-Modified to Now() to leverage caching heuristics built into modern browsers: - // https://github.com/ipfs/kubo/pull/8074#pullrequestreview-645196768 - // but we should not set it to fake values and use Cache-Control based on TTL instead - modtime = time.Now() + if ttl > 0 { + // When we know the TTL, set the Cache-Control header and disable Last-Modified. + w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", int(ttl.Seconds()))) + } - // TODO: set Cache-Control based on TTL of IPNS/DNSLink: https://github.com/ipfs/kubo/issues/1818#issuecomment-1015849462 - // TODO: set Last-Modified based on /ipns/ publishing timestamp? + if lastMod.IsZero() { + // Otherwise, we set Last-Modified to the current time to leverage caching heuristics + // built into modern browsers: https://github.com/ipfs/kubo/pull/8074#pullrequestreview-645196768 + modtime = time.Now() + } else { + modtime = lastMod + } } else { - // immutable! CACHE ALL THE THINGS, FOREVER! wolololol w.Header().Set("Cache-Control", immutableCacheControl) + modtime = noModtime // disable Last-Modified - // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) - modtime = noModtime - - // TODO: set Last-Modified? - TBD - /ipfs/ modification metadata is present in unixfs 1.5 https://github.com/ipfs/kubo/issues/6920? + // TODO: consider setting Last-Modified if UnixFS V1.5 ever gets released + // with metadata: https://github.com/ipfs/kubo/issues/6920 } return modtime diff --git a/gateway/handler_block.go b/gateway/handler_block.go index 461e306d8..e6bf2267f 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -34,7 +34,7 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h setContentDispositionHeader(w, name, "attachment") // Set remaining headers - modtime := addCacheControlHeaders(w, r, rq.contentPath, blockCid, rawResponseFormat) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, blockCid, rawResponseFormat) w.Header().Set("Content-Type", rawResponseFormat) w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 0b2162c4f..1c670b045 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -57,7 +57,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R setContentDispositionHeader(w, name, "attachment") // Set Cache-Control (same logic as for a regular files) - addCacheControlHeaders(w, r, rq.contentPath, rootCid, carResponseFormat) + addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, rootCid, carResponseFormat) // Generate the CAR Etag. etag := getCarEtag(rq.immutablePath, params, rootCid) diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index c1abb1d27..617eb9396 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -111,7 +111,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt } // Set HTTP headers (for caching, etc). Etag will be replaced if handled by serveCodecHTML. - modtime := addCacheControlHeaders(w, r, rq.contentPath, resolvedPath.RootCid(), responseContentType) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, resolvedPath.RootCid(), responseContentType) _ = setCodecContentDisposition(w, r, resolvedPath, responseContentType) w.Header().Set("Content-Type", responseContentType) w.Header().Set("X-Content-Type-Options", "nosniff") diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go index e56e6c5d9..648d51328 100644 --- a/gateway/handler_defaults.go +++ b/gateway/handler_defaults.go @@ -122,13 +122,13 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h if headResp != nil { if headResp.isFile { rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, headResp.bytesSize, headResp.startingBytes, false, true, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq, headResp.bytesSize, headResp.startingBytes, false, true, pathMetadata.ContentType) } else if headResp.isSymLink { rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, headResp.bytesSize, nil, true, true, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq, headResp.bytesSize, nil, true, true, pathMetadata.ContentType) } else if headResp.isDir { rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, true, nil, ranges, rq.begin, rq.logger) + return i.serveDirectory(ctx, w, r, resolvedPath, rq, true, nil, ranges) } } else { if getResp.bytes != nil { @@ -140,14 +140,14 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h rangeRequestStartsAtZero = false } } - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, getResp.bytesSize, getResp.bytes, false, rangeRequestStartsAtZero, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq, getResp.bytesSize, getResp.bytes, false, rangeRequestStartsAtZero, pathMetadata.ContentType) } else if getResp.symlink != nil { rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) // Note: this ignores range requests against symlinks - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, getResp.bytesSize, getResp.symlink, true, true, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq, getResp.bytesSize, getResp.symlink, true, true, pathMetadata.ContentType) } else if getResp.directoryMetadata != nil { rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, false, getResp.directoryMetadata, ranges, rq.begin, rq.logger) + return i.serveDirectory(ctx, w, r, resolvedPath, rq, false, getResp.directoryMetadata, ranges) } } diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index 6af1d0c4d..0b2634804 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -31,7 +31,7 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R rootCid := pathMetadata.LastSegment.RootCid() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, rq.contentPath, rootCid, tarResponseFormat) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, rootCid, tarResponseFormat) // Set Content-Disposition var name string diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index 6d1f40c25..0cd4d3c71 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -7,6 +7,7 @@ import ( gopath "path" "strconv" "strings" + "time" "github.com/ipfs/boxo/path" redirects "github.com/ipfs/go-ipfs-redirects-file" @@ -223,7 +224,7 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat logger.Debugf("using _redirects: custom %d file at %q", status, content4xxPath) w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) - addCacheControlHeaders(w, r, content4xxPath, content4xxCid, "") + addCacheControlHeaders(w, r, content4xxPath, 0, time.Time{}, content4xxCid, "") w.WriteHeader(status) _, err = io.CopyN(w, content4xxFile, size) return err diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index 479f4e3c9..678c51aba 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -17,13 +17,12 @@ import ( cid "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" ) // serveDirectory returns the best representation of UnixFS directory // // It will return index.html if present, or generate directory listing otherwise. -func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, isHeadRequest bool, directoryMetadata *directoryMetadata, ranges []ByteRange, begin time.Time, logger *zap.SugaredLogger) bool { +func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, rq *requestData, isHeadRequest bool, directoryMetadata *directoryMetadata, ranges []ByteRange) bool { ctx, span := spanTrace(ctx, "Handler.ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -52,14 +51,14 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } // /ipfs/cid/foo?bar must be redirected to /ipfs/cid/foo/?bar redirectURL := originalURLPath + suffix - logger.Debugw("directory location moved permanently", "status", http.StatusMovedPermanently) + rq.logger.Debugw("directory location moved permanently", "status", http.StatusMovedPermanently) http.Redirect(w, r, redirectURL, http.StatusMovedPermanently) return true } } // Check if directory has index.html, if so, serveFile - idxPath, err := path.Join(contentPath, "index.html") + idxPath, err := path.Join(rq.contentPath, "index.html") if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false @@ -113,17 +112,19 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } if err == nil { - logger.Debugw("serving index.html file", "path", idxPath) + rq.logger.Debugw("serving index.html file", "path", idxPath) + originalContentPath := rq.contentPath + rq.contentPath = idxPath // write to request - success := i.serveFile(ctx, w, r, resolvedPath, idxPath, idxFileSize, idxFileBytes, false, returnRangeStartsAtZero, "text/html", begin) + success := i.serveFile(ctx, w, r, resolvedPath, rq, idxFileSize, idxFileBytes, false, returnRangeStartsAtZero, "text/html") if success { - i.unixfsDirIndexGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsDirIndexGetMetric.WithLabelValues(originalContentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) } return success } if isErrNotFound(err) { - logger.Debugw("no index.html; noop", "path", idxPath) + rq.logger.Debugw("no index.html; noop", "path", idxPath) } else if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false @@ -137,8 +138,13 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * dirEtag := getDirListingEtag(resolvedPath.RootCid()) w.Header().Set("Etag", dirEtag) + // Add TTL if known. + if rq.ttl > 0 { + w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", int(rq.ttl.Seconds()))) + } + if r.Method == http.MethodHead { - logger.Debug("return as request's HTTP method is HEAD") + rq.logger.Debug("return as request's HTTP method is HEAD") return true } @@ -169,7 +175,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * backLink := originalURLPath // don't go further up than /ipfs/$hash/ - pathSplit := strings.Split(contentPath.String(), "/") + pathSplit := strings.Split(rq.contentPath.String(), "/") switch { // skip backlink when listing a content root case len(pathSplit) == 3: // url: /ipfs/$hash @@ -190,20 +196,20 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * size := humanize.Bytes(directoryMetadata.dagSize) hash := resolvedPath.RootCid().String() - globalData := i.getTemplateGlobalData(r, contentPath) + globalData := i.getTemplateGlobalData(r, rq.contentPath) // See comment above where originalUrlPath is declared. tplData := assets.DirectoryTemplateData{ GlobalData: globalData, Listing: dirListing, Size: size, - Path: contentPath.String(), - Breadcrumbs: assets.Breadcrumbs(contentPath.String(), globalData.DNSLink), + Path: rq.contentPath.String(), + Breadcrumbs: assets.Breadcrumbs(rq.contentPath.String(), globalData.DNSLink), BackLink: backLink, Hash: hash, } - logger.Debugw("request processed", "tplDataDNSLink", globalData.DNSLink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) + rq.logger.Debugw("request processed", "tplDataDNSLink", globalData.DNSLink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) if err := assets.DirectoryTemplate.Execute(w, tplData); err != nil { i.webError(w, r, err, http.StatusInternalServerError) @@ -211,7 +217,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } // Update metrics - i.unixfsGenDirListingGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsGenDirListingGetMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) return true } diff --git a/gateway/handler_unixfs_dir_test.go b/gateway/handler_unixfs_dir_test.go index 48f2625d6..e44708687 100644 --- a/gateway/handler_unixfs_dir_test.go +++ b/gateway/handler_unixfs_dir_test.go @@ -28,7 +28,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { k3, err := backend.resolvePathNoRootsReturned(ctx, p3) require.NoError(t, err) - backend.namesys["/ipns/example.net"] = path.FromCid(root) + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.FromCid(root), 0) // make request to directory listing req := mustNewRequest(t, http.MethodGet, ts.URL+"/foo%3F%20%23%3C%27/", nil) diff --git a/gateway/handler_unixfs_file.go b/gateway/handler_unixfs_file.go index fb1f9940c..734e11025 100644 --- a/gateway/handler_unixfs_file.go +++ b/gateway/handler_unixfs_file.go @@ -19,15 +19,15 @@ import ( // serveFile returns data behind a file along with HTTP headers based on // the file itself, its CID and the contentPath used for accessing it. -func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, fileSize int64, fileBytes io.ReadCloser, isSymlink bool, returnRangeStartsAtZero bool, fileContentType string, begin time.Time) bool { +func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, rq *requestData, fileSize int64, fileBytes io.ReadCloser, isSymlink bool, returnRangeStartsAtZero bool, fileContentType string) bool { _, span := spanTrace(ctx, "Handler.ServeFile", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.RootCid(), "") + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, resolvedPath.RootCid(), "") // Set Content-Disposition - name := addContentDispositionHeader(w, r, contentPath) + name := addContentDispositionHeader(w, r, rq.contentPath) if fileSize == 0 { // We override null files to 200 to avoid issues with fragment caching reverse proxies. @@ -89,7 +89,7 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. // Was response successful? if dataSent { // Update metrics - i.unixfsFileGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsFileGetMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) } return dataSent diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go index 3a6809555..7a8d7a170 100644 --- a/gateway/hostname_test.go +++ b/gateway/hostname_test.go @@ -20,8 +20,8 @@ func TestToSubdomainURL(t *testing.T) { testCID, err := cid.Decode("bafkqaglimvwgy3zakrsxg5cun5jxkyten5wwc2lokvjeycq") require.NoError(t, err) - backend.namesys["/ipns/dnslink.long-name.example.com"] = path.FromCid(testCID) - backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = path.FromCid(testCID) + backend.namesys["/ipns/dnslink.long-name.example.com"] = newMockNamesysItem(path.FromCid(testCID), 0) + backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = newMockNamesysItem(path.FromCid(testCID), 0) httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) diff --git a/gateway/metrics.go b/gateway/metrics.go index bccaefef2..df8639956 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -155,16 +155,16 @@ func (b *ipfsBackendWithMetrics) GetIPNSRecord(ctx context.Context, cid cid.Cid) return r, err } -func (b *ipfsBackendWithMetrics) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, error) { +func (b *ipfsBackendWithMetrics) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { begin := time.Now() name := "IPFSBackend.ResolveMutable" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) defer span.End() - p, err := b.backend.ResolveMutable(ctx, path) + p, ttl, lastMod, err := b.backend.ResolveMutable(ctx, path) b.updateBackendCallMetric(name, err, begin) - return p, err + return p, ttl, lastMod, err } func (b *ipfsBackendWithMetrics) GetDNSLinkRecord(ctx context.Context, fqdn string) (path.Path, error) { diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index bba66d94d..76359dcdd 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -11,9 +11,9 @@ import ( "regexp" "strings" "testing" + "time" "github.com/ipfs/boxo/blockservice" - nsopts "github.com/ipfs/boxo/coreiface/options/namesys" offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/namesys" @@ -51,43 +51,60 @@ func mustDo(t *testing.T, req *http.Request) *http.Response { return res } -type mockNamesys map[string]path.Path +type mockNamesysItem struct { + path path.Path + ttl time.Duration +} + +func newMockNamesysItem(p path.Path, ttl time.Duration) *mockNamesysItem { + return &mockNamesysItem{path: p, ttl: ttl} +} + +type mockNamesys map[string]*mockNamesysItem -func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...nsopts.ResolveOpt) (value path.Path, err error) { - cfg := nsopts.DefaultResolveOpts() +func (m mockNamesys) Resolve(ctx context.Context, p path.Path, opts ...namesys.ResolveOption) (result namesys.ResolveResult, err error) { + cfg := namesys.DefaultResolveOptions() for _, o := range opts { o(&cfg) } depth := cfg.Depth - if depth == nsopts.UnlimitedDepth { + if depth == namesys.UnlimitedDepth { // max uint depth = ^uint(0) } + var ( + value path.Path + ttl time.Duration + ) + name := path.SegmentsToString(p.Segments()[:2]...) for strings.HasPrefix(name, "/ipns/") { if depth == 0 { - return value, namesys.ErrResolveRecursion + return namesys.ResolveResult{Path: value, TTL: ttl}, namesys.ErrResolveRecursion } depth-- - var ok bool - value, ok = m[name] + v, ok := m[name] if !ok { - return nil, namesys.ErrResolveFailed + return namesys.ResolveResult{}, namesys.ErrResolveFailed } + value = v.path + ttl = v.ttl name = value.String() } - return value, nil + + value, err = path.Join(value, p.Segments()[2:]...) + return namesys.ResolveResult{Path: value, TTL: ttl}, err } -func (m mockNamesys) ResolveAsync(ctx context.Context, name string, opts ...nsopts.ResolveOpt) <-chan namesys.Result { - out := make(chan namesys.Result, 1) - v, err := m.Resolve(ctx, name, opts...) - out <- namesys.Result{Path: v, Err: err} +func (m mockNamesys) ResolveAsync(ctx context.Context, p path.Path, opts ...namesys.ResolveOption) <-chan namesys.ResolveAsyncResult { + out := make(chan namesys.ResolveAsyncResult, 1) + res, err := m.Resolve(ctx, p, opts...) + out <- namesys.ResolveAsyncResult{Path: res.Path, TTL: res.TTL, LastMod: res.LastMod, Err: err} close(out) return out } -func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...nsopts.PublishOption) error { +func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...namesys.PublishOption) error { return errors.New("not implemented for mockNamesys") } @@ -152,7 +169,7 @@ func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath path.ImmutableP return mb.gw.GetCAR(ctx, immutablePath, params) } -func (mb *mockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (mb *mockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { return mb.gw.ResolveMutable(ctx, p) } @@ -162,10 +179,15 @@ func (mb *mockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, er func (mb *mockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { if mb.namesys != nil { - p, err := mb.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + p, err := path.NewPath("/ipns/" + hostname) + if err != nil { + return nil, err + } + res, err := mb.namesys.Resolve(ctx, p, namesys.ResolveWithDepth(1)) if err == namesys.ErrResolveRecursion { err = nil } + p = res.Path return p, err } @@ -184,7 +206,7 @@ func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip path.P var imPath path.ImmutablePath var err error if ip.Mutable() { - imPath, err = mb.ResolveMutable(ctx, ip) + imPath, _, _, err = mb.ResolveMutable(ctx, ip) if err != nil { return path.ImmutablePath{}, err } diff --git a/namesys/base.go b/namesys/base.go deleted file mode 100644 index 6b8e41ab5..000000000 --- a/namesys/base.go +++ /dev/null @@ -1,126 +0,0 @@ -package namesys - -import ( - "context" - "strings" - "time" - - opts "github.com/ipfs/boxo/coreiface/options/namesys" - "github.com/ipfs/boxo/path" -) - -type onceResult struct { - value path.Path - ttl time.Duration - err error -} - -type resolver interface { - resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult -} - -// resolve is a helper for implementing Resolver.ResolveN using resolveOnce. -func resolve(ctx context.Context, r resolver, name string, options opts.ResolveOpts) (path.Path, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - err := ErrResolveFailed - var p path.Path - - resCh := resolveAsync(ctx, r, name, options) - - for res := range resCh { - p, err = res.Path, res.Err - if err != nil { - break - } - } - - return p, err -} - -func resolveAsync(ctx context.Context, r resolver, name string, options opts.ResolveOpts) <-chan Result { - ctx, span := StartSpan(ctx, "ResolveAsync") - defer span.End() - - resCh := r.resolveOnceAsync(ctx, name, options) - depth := options.Depth - outCh := make(chan Result, 1) - - go func() { - defer close(outCh) - ctx, span := StartSpan(ctx, "ResolveAsync.Worker") - defer span.End() - - var subCh <-chan Result - var cancelSub context.CancelFunc - defer func() { - if cancelSub != nil { - cancelSub() - } - }() - - for { - select { - case res, ok := <-resCh: - if !ok { - resCh = nil - break - } - - if res.err != nil { - emitResult(ctx, outCh, Result{Err: res.err}) - return - } - log.Debugf("resolved %s to %s", name, res.value.String()) - if !strings.HasPrefix(res.value.String(), ipnsPrefix) { - emitResult(ctx, outCh, Result{Path: res.value}) - break - } - - if depth == 1 { - emitResult(ctx, outCh, Result{Path: res.value, Err: ErrResolveRecursion}) - break - } - - subopts := options - if subopts.Depth > 1 { - subopts.Depth-- - } - - var subCtx context.Context - if cancelSub != nil { - // Cancel previous recursive resolve since it won't be used anyways - cancelSub() - } - subCtx, cancelSub = context.WithCancel(ctx) - _ = cancelSub - - p := strings.TrimPrefix(res.value.String(), ipnsPrefix) - subCh = resolveAsync(subCtx, r, p, subopts) - case res, ok := <-subCh: - if !ok { - subCh = nil - break - } - - // We don't bother returning here in case of context timeout as there is - // no good reason to do that, and we may still be able to emit a result - emitResult(ctx, outCh, res) - case <-ctx.Done(): - return - } - if resCh == nil && subCh == nil { - return - } - } - }() - return outCh -} - -func emitResult(ctx context.Context, outCh chan<- Result, r Result) { - select { - case outCh <- r: - case <-ctx.Done(): - } -} diff --git a/namesys/cache.go b/namesys/cache.go deleted file mode 100644 index 51fe3149b..000000000 --- a/namesys/cache.go +++ /dev/null @@ -1,62 +0,0 @@ -package namesys - -import ( - "time" - - "github.com/ipfs/boxo/path" -) - -func (ns *mpns) cacheGet(name string) (path.Path, bool) { - // existence of optional mapping defined via IPFS_NS_MAP is checked first - if ns.staticMap != nil { - val, ok := ns.staticMap[name] - if ok { - return val, true - } - } - - if ns.cache == nil { - return nil, false - } - - ientry, ok := ns.cache.Get(name) - if !ok { - return nil, false - } - - entry, ok := ientry.(cacheEntry) - if !ok { - // should never happen, purely for sanity - log.Panicf("unexpected type %T in cache for %q.", ientry, name) - } - - if time.Now().Before(entry.eol) { - return entry.val, true - } - - ns.cache.Remove(name) - - return nil, false -} - -func (ns *mpns) cacheSet(name string, val path.Path, ttl time.Duration) { - if ns.cache == nil || ttl <= 0 { - return - } - ns.cache.Add(name, cacheEntry{ - val: val, - eol: time.Now().Add(ttl), - }) -} - -func (ns *mpns) cacheInvalidate(name string) { - if ns.cache == nil { - return - } - ns.cache.Remove(name) -} - -type cacheEntry struct { - val path.Path - eol time.Time -} diff --git a/namesys/dns.go b/namesys/dns_resolver.go similarity index 58% rename from namesys/dns.go rename to namesys/dns_resolver.go index 48becfc1c..de5558ab6 100644 --- a/namesys/dns.go +++ b/namesys/dns_resolver.go @@ -7,9 +7,9 @@ import ( "net" gopath "path" "strings" + "time" - opts "github.com/ipfs/boxo/coreiface/options/namesys" - "github.com/ipfs/boxo/path" + path "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" dns "github.com/miekg/dns" "go.opentelemetry.io/otel/attribute" @@ -19,80 +19,65 @@ import ( // LookupTXTFunc is a function that lookups TXT record values. type LookupTXTFunc func(ctx context.Context, name string) (txt []string, err error) -// DNSResolver implements a Resolver on DNS domains +// DNSResolver implements [Resolver] on DNS domains. type DNSResolver struct { lookupTXT LookupTXTFunc - // TODO: maybe some sort of caching? - // cache would need a timeout } +var _ Resolver = &DNSResolver{} + // NewDNSResolver constructs a name resolver using DNS TXT records. func NewDNSResolver(lookup LookupTXTFunc) *DNSResolver { return &DNSResolver{lookupTXT: lookup} } -// Resolve implements Resolver. -func (r *DNSResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { - ctx, span := StartSpan(ctx, "DNSResolver.Resolve") +func (r *DNSResolver) Resolve(ctx context.Context, p path.Path, options ...ResolveOption) (ResolveResult, error) { + ctx, span := startSpan(ctx, "DNSResolver.Resolve", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - return resolve(ctx, r, name, opts.ProcessOpts(options)) + return resolve(ctx, r, p, ProcessResolveOptions(options)) } -// ResolveAsync implements Resolver. -func (r *DNSResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { - ctx, span := StartSpan(ctx, "DNSResolver.ResolveAsync") +func (r *DNSResolver) ResolveAsync(ctx context.Context, p path.Path, options ...ResolveOption) <-chan ResolveAsyncResult { + ctx, span := startSpan(ctx, "DNSResolver.ResolveAsync", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - return resolveAsync(ctx, r, name, opts.ProcessOpts(options)) -} - -type lookupRes struct { - path path.Path - error error + return resolveAsync(ctx, r, p, ProcessResolveOptions(options)) } -// resolveOnce implements resolver. -// TXT records for a given domain name should contain a b58 -// encoded multihash. -func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - ctx, span := StartSpan(ctx, "DNSResolver.ResolveOnceAsync") +func (r *DNSResolver) resolveOnceAsync(ctx context.Context, p path.Path, options ResolveOptions) <-chan ResolveAsyncResult { + ctx, span := startSpan(ctx, "DNSResolver.ResolveOnceAsync", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - var fqdn string - out := make(chan onceResult, 1) - segments := strings.SplitN(name, "/", 2) - domain := segments[0] + out := make(chan ResolveAsyncResult, 1) + if p.Namespace() != path.IPNSNamespace { + out <- ResolveAsyncResult{Err: fmt.Errorf("unsupported namespace: %s", p.Namespace())} + close(out) + return out + } - if _, ok := dns.IsDomainName(domain); !ok { - out <- onceResult{err: fmt.Errorf("not a valid domain name: %s", domain)} + fqdn := p.Segments()[1] + if _, ok := dns.IsDomainName(fqdn); !ok { + out <- ResolveAsyncResult{Err: fmt.Errorf("not a valid domain name: %q", fqdn)} close(out) return out } - log.Debugf("DNSResolver resolving %s", domain) - if strings.HasSuffix(domain, ".") { - fqdn = domain - } else { - fqdn = domain + "." + log.Debugf("DNSResolver resolving %s", fqdn) + + if !strings.HasSuffix(fqdn, ".") { + fqdn += "." } - rootChan := make(chan lookupRes, 1) + rootChan := make(chan ResolveAsyncResult, 1) go workDomain(ctx, r, fqdn, rootChan) - subChan := make(chan lookupRes, 1) + subChan := make(chan ResolveAsyncResult, 1) go workDomain(ctx, r, "_dnslink."+fqdn, subChan) - appendPath := func(p path.Path) (path.Path, error) { - if len(segments) > 1 { - return path.Join(p, segments[1]) - } - return p, nil - } - go func() { defer close(out) - ctx, span := StartSpan(ctx, "DNSResolver.ResolveOnceAsync.Worker") + ctx, span := startSpan(ctx, "DNSResolver.ResolveOnceAsync.Worker") defer span.End() var rootResErr, subResErr error @@ -103,26 +88,26 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options subChan = nil break } - if subRes.error == nil { - p, err := appendPath(subRes.path) - emitOnceResult(ctx, out, onceResult{value: p, err: err}) + if subRes.Err == nil { + p, err := joinPaths(subRes.Path, p) + emitOnceResult(ctx, out, ResolveAsyncResult{Path: p, LastMod: time.Now(), Err: err}) // Return without waiting for rootRes, since this result // (for "_dnslink."+fqdn) takes precedence return } - subResErr = subRes.error + subResErr = subRes.Err case rootRes, ok := <-rootChan: if !ok { rootChan = nil break } - if rootRes.error == nil { - p, err := appendPath(rootRes.path) - emitOnceResult(ctx, out, onceResult{value: p, err: err}) + if rootRes.Err == nil { + p, err := joinPaths(rootRes.Path, p) + emitOnceResult(ctx, out, ResolveAsyncResult{Path: p, LastMod: time.Now(), Err: err}) // Do not return here. Wait for subRes so that it is // output last if good, thereby giving subRes precedence. } else { - rootResErr = rootRes.error + rootResErr = rootRes.Err } case <-ctx.Done(): return @@ -134,8 +119,8 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options // dnslink, then output a more specific error message if rootResErr == ErrResolveFailed && subResErr == ErrResolveFailed { // Wrap error so that it can be tested if it is a ErrResolveFailed - err := fmt.Errorf("%w: _dnslink subdomain at %q is missing a TXT record (https://docs.ipfs.tech/concepts/dnslink/)", ErrResolveFailed, gopath.Base(name)) - emitOnceResult(ctx, out, onceResult{err: err}) + err := fmt.Errorf("%w: _dnslink subdomain at %q is missing a TXT record (https://docs.ipfs.tech/concepts/dnslink/)", ErrResolveFailed, gopath.Base(fqdn)) + emitOnceResult(ctx, out, ResolveAsyncResult{Err: err}) } return } @@ -145,8 +130,8 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options return out } -func workDomain(ctx context.Context, r *DNSResolver, name string, res chan lookupRes) { - ctx, span := StartSpan(ctx, "DNSResolver.WorkDomain", trace.WithAttributes(attribute.String("Name", name))) +func workDomain(ctx context.Context, r *DNSResolver, name string, res chan ResolveAsyncResult) { + ctx, span := startSpan(ctx, "DNSResolver.WorkDomain", trace.WithAttributes(attribute.String("Name", name))) defer span.End() defer close(res) @@ -161,20 +146,20 @@ func workDomain(ctx context.Context, r *DNSResolver, name string, res chan looku } } // Could not look up any text records for name - res <- lookupRes{nil, err} + res <- ResolveAsyncResult{Err: err} return } for _, t := range txt { p, err := parseEntry(t) if err == nil { - res <- lookupRes{p, nil} + res <- ResolveAsyncResult{Path: p} return } } // There were no TXT records with a dnslink - res <- lookupRes{nil, ErrResolveFailed} + res <- ResolveAsyncResult{Err: ErrResolveFailed} } func parseEntry(txt string) (path.Path, error) { diff --git a/namesys/dns_resolver_test.go b/namesys/dns_resolver_test.go new file mode 100644 index 000000000..f45020a5c --- /dev/null +++ b/namesys/dns_resolver_test.go @@ -0,0 +1,197 @@ +package namesys + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDNSParseEntry(t *testing.T) { + t.Parallel() + + t.Run("Valid entries", func(t *testing.T) { + t.Parallel() + + for _, entry := range []string{ + "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", + "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz/", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + } { + _, err := parseEntry(entry) + assert.NoError(t, err) + } + }) + + t.Run("Invalid entries", func(t *testing.T) { + t.Parallel() + + for _, entry := range []string{ + "QmYhE8xgFCjGcz6PHgnvJz5NOTCORRECT", + "quux=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=", + "dnslink=/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", + "dnslink=ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", + } { + _, err := parseEntry(entry) + assert.Error(t, err) + } + }) +} + +type mockDNS struct { + entries map[string][]string +} + +func (m *mockDNS) lookupTXT(ctx context.Context, name string) (txt []string, err error) { + txt, ok := m.entries[name] + if !ok { + return nil, fmt.Errorf("no TXT entry for %s", name) + } + return txt, nil +} + +func newMockDNS() *mockDNS { + return &mockDNS{ + entries: map[string][]string{ + "multihash.example.com.": { + "dnslink=QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "ipfs.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.dipfs.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "dns1.example.com.": { + "dnslink=/ipns/ipfs.example.com", + }, + "dns2.example.com.": { + "dnslink=/ipns/dns1.example.com", + }, + "multi.example.com.": { + "some stuff", + "dnslink=/ipns/dns1.example.com", + "masked dnslink=/ipns/example.invalid", + }, + "equals.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", + }, + "loop1.example.com.": { + "dnslink=/ipns/loop2.example.com", + }, + "loop2.example.com.": { + "dnslink=/ipns/loop1.example.com", + }, + "_dnslink.dloop1.example.com.": { + "dnslink=/ipns/loop2.example.com", + }, + "_dnslink.dloop2.example.com.": { + "dnslink=/ipns/loop1.example.com", + }, + "bad.example.com.": { + "dnslink=", + }, + "withsegment.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", + }, + "withrecsegment.example.com.": { + "dnslink=/ipns/withsegment.example.com/subsub", + }, + "withtrailing.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/", + }, + "withtrailingrec.example.com.": { + "dnslink=/ipns/withtrailing.example.com/segment/", + }, + "double.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.double.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "double.conflict.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.conflict.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", + }, + "fqdn.example.com.": { + "dnslink=/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", + }, + "en.wikipedia-on-ipfs.org.": { + "dnslink=/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", + }, + "custom.non-icann.tldextravaganza.": { + "dnslink=/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", + }, + "singlednslabelshouldbeok.": { + "dnslink=/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", + }, + "www.wealdtech.eth.": { + "dnslink=/ipns/ipfs.example.com", + }, + }, + } +} + +func TestDNSResolution(t *testing.T) { + t.Parallel() + r := &DNSResolver{lookupTXT: newMockDNS().lookupTXT} + + for _, testCase := range []struct { + name string + depth uint + expectedPath string + expectedError error + }{ + {"/ipns/multihash.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/ipfs.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/dipfs.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/dns1.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/dns1.example.com", 1, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"/ipns/dns2.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/dns2.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion}, + {"/ipns/dns2.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"/ipns/multi.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/multi.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion}, + {"/ipns/multi.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"/ipns/equals.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", nil}, + {"/ipns/loop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"/ipns/loop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"/ipns/loop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"/ipns/loop1.example.com", DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"/ipns/dloop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"/ipns/dloop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"/ipns/dloop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"/ipns/dloop1.example.com", DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"/ipns/bad.example.com", DefaultDepthLimit, "", ErrResolveFailed}, + {"/ipns/bad.example.com", DefaultDepthLimit, "", ErrResolveFailed}, + {"/ipns/withsegment.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", nil}, + {"/ipns/withrecsegment.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub", nil}, + {"/ipns/withsegment.example.com/test1", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/test1", nil}, + {"/ipns/withrecsegment.example.com/test2", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test2", nil}, + {"/ipns/withrecsegment.example.com/test3/", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test3/", nil}, + {"/ipns/withtrailingrec.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/", nil}, + {"/ipns/double.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/conflict.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", nil}, + {"/ipns/fqdn.example.com.", DefaultDepthLimit, "/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", nil}, + {"/ipns/en.wikipedia-on-ipfs.org", 2, "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", nil}, + {"/ipns/custom.non-icann.tldextravaganza.", 2, "/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", nil}, + {"/ipns/singlednslabelshouldbeok", 2, "/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", nil}, + {"/ipns/www.wealdtech.eth", 1, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"/ipns/www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + } { + t.Run(testCase.name, func(t *testing.T) { + testResolution(t, r, testCase.name, (testCase.depth), testCase.expectedPath, 0, testCase.expectedError) + }) + } +} diff --git a/namesys/dns_test.go b/namesys/dns_test.go deleted file mode 100644 index a31a53582..000000000 --- a/namesys/dns_test.go +++ /dev/null @@ -1,184 +0,0 @@ -package namesys - -import ( - "context" - "fmt" - "testing" - - opts "github.com/ipfs/boxo/coreiface/options/namesys" -) - -type mockDNS struct { - entries map[string][]string -} - -func (m *mockDNS) lookupTXT(ctx context.Context, name string) (txt []string, err error) { - txt, ok := m.entries[name] - if !ok { - return nil, fmt.Errorf("no TXT entry for %s", name) - } - return txt, nil -} - -func TestDnsEntryParsing(t *testing.T) { - goodEntries := []string{ - "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", - "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz/", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - } - - badEntries := []string{ - "QmYhE8xgFCjGcz6PHgnvJz5NOTCORRECT", - "quux=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=", - "dnslink=/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", - "dnslink=ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", - } - - for _, e := range goodEntries { - _, err := parseEntry(e) - if err != nil { - t.Log("expected entry to parse correctly!") - t.Log(e) - t.Fatal(err) - } - } - - for _, e := range badEntries { - _, err := parseEntry(e) - if err == nil { - t.Log("expected entry parse to fail!") - t.Fatal(err) - } - } -} - -func newMockDNS() *mockDNS { - return &mockDNS{ - entries: map[string][]string{ - "multihash.example.com.": { - "dnslink=QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "ipfs.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "_dnslink.dipfs.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "dns1.example.com.": { - "dnslink=/ipns/ipfs.example.com", - }, - "dns2.example.com.": { - "dnslink=/ipns/dns1.example.com", - }, - "multi.example.com.": { - "some stuff", - "dnslink=/ipns/dns1.example.com", - "masked dnslink=/ipns/example.invalid", - }, - "equals.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", - }, - "loop1.example.com.": { - "dnslink=/ipns/loop2.example.com", - }, - "loop2.example.com.": { - "dnslink=/ipns/loop1.example.com", - }, - "_dnslink.dloop1.example.com.": { - "dnslink=/ipns/loop2.example.com", - }, - "_dnslink.dloop2.example.com.": { - "dnslink=/ipns/loop1.example.com", - }, - "bad.example.com.": { - "dnslink=", - }, - "withsegment.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", - }, - "withrecsegment.example.com.": { - "dnslink=/ipns/withsegment.example.com/subsub", - }, - "withtrailing.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/", - }, - "withtrailingrec.example.com.": { - "dnslink=/ipns/withtrailing.example.com/segment/", - }, - "double.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "_dnslink.double.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "double.conflict.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "_dnslink.conflict.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", - }, - "fqdn.example.com.": { - "dnslink=/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", - }, - "en.wikipedia-on-ipfs.org.": { - "dnslink=/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", - }, - "custom.non-icann.tldextravaganza.": { - "dnslink=/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", - }, - "singlednslabelshouldbeok.": { - "dnslink=/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", - }, - "www.wealdtech.eth.": { - "dnslink=/ipns/ipfs.example.com", - }, - }, - } -} - -func TestDNSResolution(t *testing.T) { - mock := newMockDNS() - r := &DNSResolver{lookupTXT: mock.lookupTXT} - testResolution(t, r, "multihash.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "ipfs.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dipfs.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns1.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns1.example.com", 1, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "dns2.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns2.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion) - testResolution(t, r, "dns2.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "multi.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "multi.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion) - testResolution(t, r, "multi.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "equals.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", nil) - testResolution(t, r, "loop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", opts.DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", opts.DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "bad.example.com", opts.DefaultDepthLimit, "", ErrResolveFailed) - testResolution(t, r, "withsegment.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", nil) - testResolution(t, r, "withrecsegment.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub", nil) - testResolution(t, r, "withsegment.example.com/test1", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/test1", nil) - testResolution(t, r, "withrecsegment.example.com/test2", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test2", nil) - testResolution(t, r, "withrecsegment.example.com/test3/", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test3/", nil) - testResolution(t, r, "withtrailingrec.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/", nil) - testResolution(t, r, "double.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "conflict.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", nil) - testResolution(t, r, "fqdn.example.com.", opts.DefaultDepthLimit, "/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", nil) - testResolution(t, r, "en.wikipedia-on-ipfs.org", 2, "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", nil) - testResolution(t, r, "custom.non-icann.tldextravaganza.", 2, "/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", nil) - testResolution(t, r, "singlednslabelshouldbeok", 2, "/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", nil) - testResolution(t, r, "www.wealdtech.eth", 1, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) -} diff --git a/namesys/interface.go b/namesys/interface.go index 5d50936ee..7215d2669 100644 --- a/namesys/interface.go +++ b/namesys/interface.go @@ -1,52 +1,50 @@ -/* -Package namesys implements resolvers and publishers for the IPFS -naming system (IPNS). - -The core of IPFS is an immutable, content-addressable Merkle graph. -That works well for many use cases, but doesn't allow you to answer -questions like "what is Alice's current homepage?". The mutable name -system allows Alice to publish information like: - - The current homepage for alice.example.com is - /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj - -or: - - The current homepage for node - QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy - is - /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj - -The mutable name system also allows users to resolve those references -to find the immutable IPFS object currently referenced by a given -mutable name. - -For command-line bindings to this functionality, see: - - ipfs name - ipfs dns - ipfs resolve -*/ package namesys import ( "context" "errors" + "time" - opts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" + logging "github.com/ipfs/go-log/v2" ci "github.com/libp2p/go-libp2p/core/crypto" ) -// ErrResolveFailed signals an error when attempting to resolve. -var ErrResolveFailed = errors.New("could not resolve name") +var log = logging.Logger("namesys") -// ErrResolveRecursion signals a recursion-depth limit. -var ErrResolveRecursion = errors.New( - "could not resolve name (recursion limit exceeded)") +var ( + // ErrResolveFailed signals an error when attempting to resolve. + ErrResolveFailed = errors.New("could not resolve name") -// ErrPublishFailed signals an error when attempting to publish. -var ErrPublishFailed = errors.New("could not publish name") + // ErrResolveRecursion signals a recursion-depth limit. + ErrResolveRecursion = errors.New("could not resolve name (recursion limit exceeded)") + + // ErrNoNamesys is an explicit error for when no [NameSystem] is provided. + ErrNoNamesys = errors.New("no namesys has been provided") +) + +const ( + // DefaultDepthLimit is the default depth limit used by [Resolver]. + DefaultDepthLimit = 32 + + // UnlimitedDepth allows infinite recursion in [Resolver]. You probably don't want + // to use this, but it's here if you absolutely trust resolution to eventually + // complete and can't put an upper limit on how many steps it will take. + UnlimitedDepth = 0 + + // DefaultIPNSRecordTTL specifies the time that the record can be cached before + // checking if its validity again. + DefaultIPNSRecordTTL = time.Minute + + // DefaultIPNSRecordEOL specifies the time that the network will cache IPNS + // records after being published. Records should be re-published before this + // interval expires. We use the same default expiration as the DHT. + DefaultIPNSRecordEOL = 48 * time.Hour + + // DefaultResolverCacheTTL defines max TTL of a record placed in [NameSystem] cache. + DefaultResolverCacheTTL = time.Minute +) // NameSystem represents a cohesive name publishing and resolving system. // @@ -60,39 +58,168 @@ type NameSystem interface { Publisher } -// Result is the return type for Resolver.ResolveAsync. -type Result struct { - Path path.Path - Err error +// ResolveResult is the return type for [Resolver.Resolve]. +type ResolveResult struct { + Path path.Path + TTL time.Duration + LastMod time.Time +} + +// ResolveAsyncResult is the return type for [Resolver.ResolveAsync]. +type ResolveAsyncResult struct { + Path path.Path + TTL time.Duration + LastMod time.Time + Err error } // Resolver is an object capable of resolving names. type Resolver interface { - // Resolve performs a recursive lookup, returning the dereferenced - // path. For example, if ipfs.io has a DNS TXT record pointing to + // Resolve performs a recursive lookup, returning the dereferenced path and the TTL. + // If the TTL is unknown, then a TTL of 0 is returned. For example, if example.com + // has a DNS TXT record pointing to: + // // /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + // // and there is a DHT IPNS entry for + // // QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy // -> /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj + // // then + // // Resolve(ctx, "/ipns/ipfs.io") + // // will resolve both names, returning + // // /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj // - // There is a default depth-limit to avoid infinite recursion. Most - // users will be fine with this default limit, but if you need to - // adjust the limit you can specify it as an option. - Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (value path.Path, err error) + // There is a default depth-limit to avoid infinite recursion. Most users will be fine with + // this default limit, but if you need to adjust the limit you can specify it as an option. + Resolve(context.Context, path.Path, ...ResolveOption) (ResolveResult, error) + + // ResolveAsync performs recursive name lookup, like Resolve, but it returns entries as + // they are discovered in the DHT. Each returned result is guaranteed to be "better" + // (which usually means newer) than the previous one. + ResolveAsync(context.Context, path.Path, ...ResolveOption) <-chan ResolveAsyncResult +} + +// ResolveOptions specifies options for resolving an IPNS Path. +type ResolveOptions struct { + // Depth is the recursion depth limit. + Depth uint + + // DhtRecordCount is the number of IPNS Records to retrieve from the DHT + // (the best record is selected from this set). + DhtRecordCount uint - // ResolveAsync performs recursive name lookup, like Resolve, but it returns - // entries as they are discovered in the DHT. Each returned result is guaranteed - // to be "better" (which usually means newer) than the previous one. - ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result + // DhtTimeout is the amount of time to wait for DHT records to be fetched + // and verified. A zero value indicates that there is no explicit timeout + // (although there is an implicit timeout due to dial timeouts within the DHT). + DhtTimeout time.Duration +} + +// DefaultResolveOptions returns the default options for resolving an IPNS Path. +func DefaultResolveOptions() ResolveOptions { + return ResolveOptions{ + Depth: DefaultDepthLimit, + DhtRecordCount: 16, + DhtTimeout: time.Minute, + } +} + +// ResolveOption is used to set a resolve option. +type ResolveOption func(*ResolveOptions) + +// ResolveWithDepth sets [ResolveOptions.Depth]. +func ResolveWithDepth(depth uint) ResolveOption { + return func(o *ResolveOptions) { + o.Depth = depth + } +} + +// ResolveWithDhtRecordCount sets [ResolveOptions.DhtRecordCount]. +func ResolveWithDhtRecordCount(count uint) ResolveOption { + return func(o *ResolveOptions) { + o.DhtRecordCount = count + } +} + +// ResolveWithDhtTimeout sets [ResolveOptions.ResolveWithDhtTimeout]. +func ResolveWithDhtTimeout(timeout time.Duration) ResolveOption { + return func(o *ResolveOptions) { + o.DhtTimeout = timeout + } +} + +// ProcessResolveOptions converts an array of [ResolveOption] into a [ResolveOptions] object. +func ProcessResolveOptions(opts []ResolveOption) ResolveOptions { + resolveOptions := DefaultResolveOptions() + for _, option := range opts { + option(&resolveOptions) + } + return resolveOptions } // Publisher is an object capable of publishing particular names. type Publisher interface { - // Publish establishes a name-value mapping. - // TODO make this not PrivKey specific. - Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...opts.PublishOption) error + // Publish publishes the given value under the name represented by the given private key. + Publish(ctx context.Context, sk ci.PrivKey, value path.Path, options ...PublishOption) error +} + +// PublishOptions specifies options for publishing an IPNS Record. +type PublishOptions struct { + // EOL defines for how long the published value is valid. + EOL time.Time + + // TTL defines for how long the published value is valid. + TTL time.Duration + + // IPNSOptions are options passed by [IPNSPublisher] to [ipns.NewRecord] when + // creating a new record to publish. With this options, you can further customize + // the way IPNS Records are created. + IPNSOptions []ipns.Option +} + +// DefaultPublishOptions returns the default options for publishing an IPNS Record. +func DefaultPublishOptions() PublishOptions { + return PublishOptions{ + EOL: time.Now().Add(DefaultIPNSRecordEOL), + TTL: DefaultIPNSRecordTTL, + } +} + +// PublishOption is used to set an option for [PublishOptions]. +type PublishOption func(*PublishOptions) + +// PublishWithEOL sets [PublishOptions.EOL]. +func PublishWithEOL(eol time.Time) PublishOption { + return func(o *PublishOptions) { + o.EOL = eol + } +} + +// PublishWithEOL sets [PublishOptions.TTL]. +func PublishWithTTL(ttl time.Duration) PublishOption { + return func(o *PublishOptions) { + o.TTL = ttl + } +} + +// PublishWithIPNSOption adds an [ipns.Option] to [PublishOptions.IPNSOptions]. +// These options are used by [IPNSPublisher], which passes them onto the IPNS +// record creation at [ipns.NewRecord] +func PublishWithIPNSOption(option ipns.Option) PublishOption { + return func(o *PublishOptions) { + o.IPNSOptions = append(o.IPNSOptions, option) + } +} + +// ProcessPublishOptions converts an array of [PublishOption] into a [PublishOptions] object. +func ProcessPublishOptions(opts []PublishOption) PublishOptions { + publishOptions := DefaultPublishOptions() + for _, option := range opts { + option(&publishOptions) + } + return publishOptions } diff --git a/namesys/publisher.go b/namesys/ipns_publisher.go similarity index 51% rename from namesys/publisher.go rename to namesys/ipns_publisher.go index 9cb3ae66a..7ac2e0dde 100644 --- a/namesys/publisher.go +++ b/namesys/ipns_publisher.go @@ -2,11 +2,11 @@ package namesys import ( "context" + "errors" "strings" "sync" "time" - opts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" ds "github.com/ipfs/go-datastore" @@ -19,40 +19,39 @@ import ( "go.opentelemetry.io/otel/trace" ) -const ipnsPrefix = "/ipns/" - -// IpnsPublisher is capable of publishing and resolving names to the IPFS -// routing system. -type IpnsPublisher struct { +// IPNSPublisher implements [Publisher] for IPNS Records. +type IPNSPublisher struct { routing routing.ValueStore ds ds.Datastore - // Used to ensure we assign IPNS records *sequential* sequence numbers. + // Used to ensure we assign IPNS records sequential sequence numbers. mu sync.Mutex } -// NewIpnsPublisher constructs a publisher for the IPFS Routing name system. -func NewIpnsPublisher(route routing.ValueStore, ds ds.Datastore) *IpnsPublisher { +var _ Publisher = &IPNSPublisher{} + +// NewIPNSResolver constructs a new [IPNSResolver] from a [routing.ValueStore] and +// a [ds.Datastore]. +func NewIPNSPublisher(route routing.ValueStore, ds ds.Datastore) *IPNSPublisher { if ds == nil { panic("nil datastore") } - return &IpnsPublisher{routing: route, ds: ds} + + return &IPNSPublisher{routing: route, ds: ds} } -// Publish implements Publisher. Accepts a keypair and a value, -// and publishes it out to the routing system -func (p *IpnsPublisher) Publish(ctx context.Context, k crypto.PrivKey, value path.Path, options ...opts.PublishOption) error { +func (p *IPNSPublisher) Publish(ctx context.Context, priv crypto.PrivKey, value path.Path, options ...PublishOption) error { log.Debugf("Publish %s", value) - ctx, span := StartSpan(ctx, "IpnsPublisher.Publish", trace.WithAttributes(attribute.String("Value", value.String()))) + ctx, span := startSpan(ctx, "IpnsPublisher.Publish", trace.WithAttributes(attribute.String("Value", value.String()))) defer span.End() - record, err := p.updateRecord(ctx, k, value, options...) + record, err := p.updateRecord(ctx, priv, value, options...) if err != nil { return err } - return PutRecordToRouting(ctx, p.routing, k.GetPublic(), record) + return PublishIPNSRecord(ctx, p.routing, priv.GetPublic(), record) } // IpnsDsKey returns a datastore key given an IPNS identifier (peer @@ -66,9 +65,9 @@ func IpnsDsKey(id peer.ID) ds.Key { // // This method will not search the routing system for records published by other // nodes. -func (p *IpnsPublisher) ListPublished(ctx context.Context) (map[peer.ID]*ipns.Record, error) { +func (p *IPNSPublisher) ListPublished(ctx context.Context) (map[peer.ID]*ipns.Record, error) { query, err := p.ds.Query(ctx, dsquery.Query{ - Prefix: ipnsPrefix, + Prefix: ipns.NamespacePrefix, }) if err != nil { return nil, err @@ -91,11 +90,11 @@ func (p *IpnsPublisher) ListPublished(ctx context.Context) (map[peer.ID]*ipns.Re log.Error("found an invalid IPNS entry:", err) continue } - if !strings.HasPrefix(result.Key, ipnsPrefix) { - log.Errorf("datastore query for keys with prefix %s returned a key: %s", ipnsPrefix, result.Key) + if !strings.HasPrefix(result.Key, ipns.NamespacePrefix) { + log.Errorf("datastore query for keys with prefix %s returned a key: %s", ipns.NamespacePrefix, result.Key) continue } - k := result.Key[len(ipnsPrefix):] + k := result.Key[len(ipns.NamespacePrefix):] pid, err := base32.RawStdEncoding.DecodeString(k) if err != nil { log.Errorf("ipns ds key invalid: %s", result.Key) @@ -113,7 +112,7 @@ func (p *IpnsPublisher) ListPublished(ctx context.Context) (map[peer.ID]*ipns.Re // // If `checkRouting` is true and we have no existing record, this method will // check the routing system for any existing records. -func (p *IpnsPublisher) GetPublished(ctx context.Context, id peer.ID, checkRouting bool) (*ipns.Record, error) { +func (p *IPNSPublisher) GetPublished(ctx context.Context, id peer.ID, checkRouting bool) (*ipns.Record, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() @@ -124,8 +123,8 @@ func (p *IpnsPublisher) GetPublished(ctx context.Context, id peer.ID, checkRouti if !checkRouting { return nil, nil } - ipnskey := string(ipns.NameFromPeer(id).RoutingKey()) - value, err = p.routing.GetValue(ctx, ipnskey) + routingKey := ipns.NameFromPeer(id).RoutingKey() + value, err = p.routing.GetValue(ctx, string(routingKey)) if err != nil { // Not found or other network issue. Can't really do // anything about this case. @@ -142,7 +141,7 @@ func (p *IpnsPublisher) GetPublished(ctx context.Context, id peer.ID, checkRouti return ipns.UnmarshalRecord(value) } -func (p *IpnsPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, value path.Path, options ...opts.PublishOption) (*ipns.Record, error) { +func (p *IPNSPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, value path.Path, options ...PublishOption) (*ipns.Record, error) { id, err := peer.IDFromPrivateKey(k) if err != nil { return nil, err @@ -157,9 +156,9 @@ func (p *IpnsPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, valu return nil, err } - seqno := uint64(0) + seq := uint64(0) if rec != nil { - seqno, err = rec.Sequence() + seq, err = rec.Sequence() if err != nil { return nil, err } @@ -171,14 +170,14 @@ func (p *IpnsPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, valu if value.String() != p.String() { // Don't bother incrementing the sequence number unless the // value changes. - seqno++ + seq++ } } - opts := opts.ProcessPublishOptions(options) + opts := ProcessPublishOptions(options) // Create record - r, err := ipns.NewRecord(k, value, seqno, opts.EOL, opts.TTL, ipns.WithV1Compatibility(opts.CompatibleWithV1)) + r, err := ipns.NewRecord(k, value, seq, opts.EOL, opts.TTL, opts.IPNSOptions...) if err != nil { return nil, err } @@ -189,21 +188,22 @@ func (p *IpnsPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, valu } // Put the new record. - key := IpnsDsKey(id) - if err := p.ds.Put(ctx, key, data); err != nil { + dsKey := IpnsDsKey(id) + if err := p.ds.Put(ctx, dsKey, data); err != nil { return nil, err } - if err := p.ds.Sync(ctx, key); err != nil { + if err := p.ds.Sync(ctx, dsKey); err != nil { return nil, err } + return r, nil } -// PutRecordToRouting publishes the given entry using the provided ValueStore, -// keyed on the ID associated with the provided public key. The public key is -// also made available to the routing system so that entries can be verified. -func PutRecordToRouting(ctx context.Context, r routing.ValueStore, k crypto.PubKey, rec *ipns.Record) error { - ctx, span := StartSpan(ctx, "PutRecordToRouting") +// PublishIPNSRecord publishes the given [ipns.Record] for the provided [crypto.PubKey] in +// the provided [routing.ValueStore]. The public key is also made available to the routing +// system if it cannot be derived from the corresponding [peer.ID]. +func PublishIPNSRecord(ctx context.Context, r routing.ValueStore, pubKey crypto.PubKey, rec *ipns.Record) error { + ctx, span := startSpan(ctx, "PutRecordToRouting") defer span.End() ctx, cancel := context.WithCancel(ctx) @@ -211,25 +211,22 @@ func PutRecordToRouting(ctx context.Context, r routing.ValueStore, k crypto.PubK errs := make(chan error, 2) // At most two errors (IPNS, and public key) - id, err := peer.IDFromPublicKey(k) + pid, err := peer.IDFromPublicKey(pubKey) if err != nil { return err } go func() { - errs <- PublishEntry(ctx, r, string(ipns.NameFromPeer(id).RoutingKey()), rec) + errs <- PutIPNSRecord(ctx, r, ipns.NameFromPeer(pid), rec) }() - // Publish the public key if a public key cannot be extracted from the ID - // TODO: once v0.4.16 is widespread enough, we can stop doing this - // and at that point we can even deprecate the /pk/ namespace in the dht - // - // NOTE: This check actually checks if the public key has been embedded - // in the IPNS entry. This check is sufficient because we embed the - // public key in the IPNS entry if it can't be extracted from the ID. - if _, err := rec.PubKey(); err == nil { + // Publish the public key if the public key cannot be extracted from the peer ID. + // This is most likely not necessary since IPNS Records include, by default, the public + // key in those cases. However, this ensures it's still possible to easily retrieve + // the public key if, for some reason, it is not embedded. + if _, err := pid.ExtractPublicKey(); errors.Is(err, peer.ErrNoPublicKey) { go func() { - errs <- PublishPublicKey(ctx, r, PkKeyForID(id), k) + errs <- PutPublicKey(ctx, r, pid, pubKey) }() if err := waitOnErrChan(ctx, errs); err != nil { @@ -249,39 +246,37 @@ func waitOnErrChan(ctx context.Context, errs chan error) error { } } -// PublishPublicKey stores the given public key in the ValueStore with the -// given key. -func PublishPublicKey(ctx context.Context, r routing.ValueStore, k string, pubk crypto.PubKey) error { - ctx, span := StartSpan(ctx, "PublishPublicKey", trace.WithAttributes(attribute.String("Key", k))) +// PutPublicKey puts the given [crypto.PubKey] for the given [peer.ID] in the [routing.ValueStore]. +func PutPublicKey(ctx context.Context, r routing.ValueStore, pid peer.ID, pubKey crypto.PubKey) error { + routingKey := PkRoutingKey(pid) + ctx, span := startSpan(ctx, "PublishPublicKey", trace.WithAttributes(attribute.String("Key", routingKey))) defer span.End() - log.Debugf("Storing pubkey at: %s", k) - pkbytes, err := crypto.MarshalPublicKey(pubk) + bytes, err := crypto.MarshalPublicKey(pubKey) if err != nil { return err } - // Store associated public key - return r.PutValue(ctx, k, pkbytes) + log.Debugf("Storing public key at: %x", routingKey) + return r.PutValue(ctx, routingKey, bytes) +} + +// PkRoutingKey returns the public key routing key for the given [peer.ID]. +func PkRoutingKey(id peer.ID) string { + return "/pk/" + string(id) } -// PublishEntry stores the given IpnsEntry in the ValueStore with the given -// ipnskey. -func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec *ipns.Record) error { - ctx, span := StartSpan(ctx, "PublishEntry", trace.WithAttributes(attribute.String("IPNSKey", ipnskey))) +// PutIPNSRecord puts the given [ipns.Record] for the given [ipns.Name] in the [routing.ValueStore]. +func PutIPNSRecord(ctx context.Context, r routing.ValueStore, name ipns.Name, rec *ipns.Record) error { + routingKey := string(name.RoutingKey()) + ctx, span := startSpan(ctx, "PublishEntry", trace.WithAttributes(attribute.String("IPNSKey", routingKey))) defer span.End() - data, err := ipns.MarshalRecord(rec) + bytes, err := ipns.MarshalRecord(rec) if err != nil { return err } - log.Debugf("Storing ipns entry at: %x", ipnskey) - // Store ipns entry at "/ipns/"+h(pubkey) - return r.PutValue(ctx, ipnskey, data) -} - -// PkKeyForID returns the public key routing key for the given peer ID. -func PkKeyForID(id peer.ID) string { - return "/pk/" + string(id) + log.Debugf("Storing ipns record at: %x", routingKey) + return r.PutValue(ctx, routingKey, bytes) } diff --git a/namesys/ipns_publisher_test.go b/namesys/ipns_publisher_test.go new file mode 100644 index 000000000..e60c6bb6e --- /dev/null +++ b/namesys/ipns_publisher_test.go @@ -0,0 +1,121 @@ +package namesys + +import ( + "context" + "crypto/rand" + "testing" + "time" + + "github.com/ipfs/boxo/path" + "github.com/stretchr/testify/require" + + dshelp "github.com/ipfs/boxo/datastore/dshelp" + "github.com/ipfs/boxo/ipns" + mockrouting "github.com/ipfs/boxo/routing/mock" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + testutil "github.com/libp2p/go-libp2p-testing/net" + ci "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" +) + +func TestIPNSPublisher(t *testing.T) { + t.Parallel() + + test := func(t *testing.T, keyType int, expectedErr error, expectedExistence bool) { + ctx := context.Background() + + // Create test identity + privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPublicKey(pubKey) + require.NoError(t, err) + + // Create IPNS Record + value, err := path.NewPath("/ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + require.NoError(t, err) + rec, err := ipns.NewRecord(privKey, value, 0, time.Now().Add(24*time.Hour), 0) + require.NoError(t, err) + + // Routing value store + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + serv := mockrouting.NewServer() + r := serv.ClientWithDatastore(context.Background(), testutil.NewIdentity(pid, testutil.ZeroLocalTCPAddress, privKey, pubKey), dstore) + + // Publish IPNS Record + err = PublishIPNSRecord(ctx, r, pubKey, rec) + require.NoError(t, err) + + // Check if IPNS Record is stored in value store + _, err = r.GetValue(ctx, string(ipns.NameFromPeer(pid).RoutingKey())) + require.NoError(t, err) + + key := dshelp.NewKeyFromBinary(ipns.NameFromPeer(pid).RoutingKey()) + exists, err := dstore.Has(ctx, key) + require.NoError(t, err) + require.True(t, exists) + + // Check for Public Key is stored in value store + pkRoutingKey := PkRoutingKey(pid) + _, err = r.GetValue(ctx, pkRoutingKey) + require.ErrorIs(t, err, expectedErr) + + // Check if Public Key is in data store for completeness + key = dshelp.NewKeyFromBinary([]byte(pkRoutingKey)) + exists, err = dstore.Has(ctx, key) + require.NoError(t, err) + require.Equal(t, expectedExistence, exists) + } + + t.Run("RSA", func(t *testing.T) { + t.Parallel() + test(t, ci.RSA, nil, true) + }) + + t.Run("Ed22519", func(t *testing.T) { + t.Parallel() + test(t, ci.Ed25519, ds.ErrNotFound, false) + }) +} + +func TestAsyncDS(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rt := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t)) + ds := &checkSyncDS{ + Datastore: ds.NewMapDatastore(), + syncKeys: make(map[ds.Key]struct{}), + } + publisher := NewIPNSPublisher(rt, ds) + + ipnsFakeID := testutil.RandIdentityOrFatal(t) + ipnsVal, err := path.NewPath("/ipns/foo.bar") + require.NoError(t, err) + + err = publisher.Publish(ctx, ipnsFakeID.PrivateKey(), ipnsVal) + require.NoError(t, err) + + ipnsKey := IpnsDsKey(ipnsFakeID.ID()) + + for k := range ds.syncKeys { + if k.IsAncestorOf(ipnsKey) || k.Equal(ipnsKey) { + return + } + } + + t.Fatal("ipns key not synced") +} + +type checkSyncDS struct { + ds.Datastore + syncKeys map[ds.Key]struct{} +} + +func (d *checkSyncDS) Sync(ctx context.Context, prefix ds.Key) error { + d.syncKeys[prefix] = struct{}{} + return d.Datastore.Sync(ctx, prefix) +} diff --git a/namesys/ipns_resolver.go b/namesys/ipns_resolver.go new file mode 100644 index 000000000..a986ff173 --- /dev/null +++ b/namesys/ipns_resolver.go @@ -0,0 +1,158 @@ +package namesys + +import ( + "context" + "fmt" + "time" + + "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" + + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p/core/routing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// IPNSResolver implements [Resolver] for IPNS Records. This resolver always returns +// a TTL if the record is still valid. It happens as follows: +// +// 1. Provisory TTL is chosen: record TTL if it exists, otherwise [DefaultIPNSRecordTTL]. +// 2. If provisory TTL expires before EOL, then returned TTL is duration between EOL and now. +// 3. If record is expired, 0 is returned as TTL. +type IPNSResolver struct { + routing routing.ValueStore +} + +var _ Resolver = &IPNSResolver{} + +// NewIPNSResolver constructs a new [IPNSResolver] from a [routing.ValueStore]. +func NewIPNSResolver(route routing.ValueStore) *IPNSResolver { + if route == nil { + panic("attempt to create resolver with nil routing system") + } + + return &IPNSResolver{ + routing: route, + } +} + +func (r *IPNSResolver) Resolve(ctx context.Context, p path.Path, options ...ResolveOption) (ResolveResult, error) { + ctx, span := startSpan(ctx, "IPNSResolver.Resolve", trace.WithAttributes(attribute.Stringer("Path", p))) + defer span.End() + + return resolve(ctx, r, p, ProcessResolveOptions(options)) +} + +func (r *IPNSResolver) ResolveAsync(ctx context.Context, p path.Path, options ...ResolveOption) <-chan ResolveAsyncResult { + ctx, span := startSpan(ctx, "IPNSResolver.ResolveAsync", trace.WithAttributes(attribute.Stringer("Path", p))) + defer span.End() + + return resolveAsync(ctx, r, p, ProcessResolveOptions(options)) +} + +func (r *IPNSResolver) resolveOnceAsync(ctx context.Context, p path.Path, options ResolveOptions) <-chan ResolveAsyncResult { + ctx, span := startSpan(ctx, "IPNSResolver.ResolveOnceAsync", trace.WithAttributes(attribute.Stringer("Path", p))) + defer span.End() + + out := make(chan ResolveAsyncResult, 1) + if p.Namespace() != path.IPNSNamespace { + out <- ResolveAsyncResult{Err: fmt.Errorf("unsupported namespace: %s", p.Namespace())} + close(out) + return out + } + + cancel := func() {} + if options.DhtTimeout != 0 { + // Resolution must complete within the timeout + ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout) + } + + name, err := ipns.NameFromString(p.Segments()[1]) + if err != nil { + out <- ResolveAsyncResult{Err: err} + close(out) + cancel() + return out + } + + vals, err := r.routing.SearchValue(ctx, string(name.RoutingKey()), dht.Quorum(int(options.DhtRecordCount))) + if err != nil { + out <- ResolveAsyncResult{Err: err} + close(out) + cancel() + return out + } + + go func() { + defer cancel() + defer close(out) + ctx, span := startSpan(ctx, "IpnsResolver.ResolveOnceAsync.Worker") + defer span.End() + + for { + select { + case val, ok := <-vals: + if !ok { + return + } + + rec, err := ipns.UnmarshalRecord(val) + if err != nil { + emitOnceResult(ctx, out, ResolveAsyncResult{Err: err}) + return + } + + resolvedBase, err := rec.Value() + if err != nil { + emitOnceResult(ctx, out, ResolveAsyncResult{Err: err}) + return + } + + resolvedBase, err = joinPaths(resolvedBase, p) + if err != nil { + emitOnceResult(ctx, out, ResolveAsyncResult{Err: err}) + return + } + + ttl, err := calculateBestTTL(rec) + if err != nil { + emitOnceResult(ctx, out, ResolveAsyncResult{Err: err}) + return + } + + // TODO: in the future it would be interesting to set the last modified date + // as the date in which the record has been signed. + emitOnceResult(ctx, out, ResolveAsyncResult{Path: resolvedBase, TTL: ttl, LastMod: time.Now()}) + case <-ctx.Done(): + return + } + } + }() + + return out +} + +func calculateBestTTL(rec *ipns.Record) (time.Duration, error) { + ttl := DefaultResolverCacheTTL + if recordTTL, err := rec.TTL(); err == nil { + ttl = recordTTL + } + + switch eol, err := rec.Validity(); err { + case ipns.ErrUnrecognizedValidity: + // No EOL. + case nil: + ttEol := time.Until(eol) + if ttEol < 0 { + // It *was* valid when we first resolved it. + ttl = 0 + } else if ttEol < ttl { + ttl = ttEol + } + default: + return 0, err + } + + return ttl, nil +} diff --git a/namesys/ipns_resolver_test.go b/namesys/ipns_resolver_test.go new file mode 100644 index 000000000..c7dcd3b97 --- /dev/null +++ b/namesys/ipns_resolver_test.go @@ -0,0 +1,131 @@ +package namesys + +import ( + "context" + "testing" + "time" + + ipns "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/routing/offline" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + record "github.com/libp2p/go-libp2p-record" + tnet "github.com/libp2p/go-libp2p-testing/net" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/stretchr/testify/require" +) + +type noFailValidator struct{} + +func (v noFailValidator) Validate(key string, value []byte) error { + return nil +} + +func (v noFailValidator) Select(key string, values [][]byte) (int, error) { + return 0, nil +} + +func TestResolver(t *testing.T) { + t.Parallel() + + pathCat := path.FromCid(cid.MustParse("bafkqabddmf2au")) + pathDog := path.FromCid(cid.MustParse("bafkqabden5tqu")) + + makeResolverDependencies := func() (tnet.Identity, ipns.Name, ds.Datastore, routing.ValueStore) { + ds := dssync.MutexWrap(ds.NewMapDatastore()) + id := tnet.RandIdentityOrFatal(t) + r := offline.NewOfflineRouter(ds, record.NamespacedValidator{ + "ipns": ipns.Validator{}, // No need for KeyBook, as records created by NameSys include PublicKey for RSA. + "pk": record.PublicKeyValidator{}, + }) + + return id, ipns.NameFromPeer(id.ID()), ds, r + } + + t.Run("Publish and resolve", func(t *testing.T) { + t.Parallel() + + id, name, ds, r := makeResolverDependencies() + resolver := NewIPNSResolver(r) + publisher := NewIPNSPublisher(r, ds) + + err := publisher.Publish(context.Background(), id.PrivateKey(), pathCat) + require.NoError(t, err) + + res, err := resolver.Resolve(context.Background(), name.AsPath()) + require.NoError(t, err) + require.Equal(t, pathCat, res.Path) + }) + + t.Run("Resolve does not return expired record", func(t *testing.T) { + t.Parallel() + + id, name, ds, r := makeResolverDependencies() + resolver := NewIPNSResolver(r) + + // Create a "bad" publisher that allows to publish expired records. + publisher := NewIPNSPublisher(offline.NewOfflineRouter(ds, record.NamespacedValidator{ + "ipns": noFailValidator{}, + "pk": record.PublicKeyValidator{}, + }), ds) + + // Publish expired. + eol := time.Now().Add(time.Hour * -1) + err := publisher.Publish(context.Background(), id.PrivateKey(), pathCat, PublishWithEOL(eol)) + require.NoError(t, err) + + // Expect to not be able to resolve. + _, err = resolver.Resolve(context.Background(), name.AsPath()) + require.ErrorIs(t, err, ErrResolveFailed) + }) + + t.Run("Resolve prefers non-expired record", func(t *testing.T) { + t.Parallel() + + id, name, ds, r := makeResolverDependencies() + resolver := NewIPNSResolver(r) + + // Create a "bad" publisher that allows to publish expired records. + publisher := NewIPNSPublisher(offline.NewOfflineRouter(ds, record.NamespacedValidator{ + "ipns": noFailValidator{}, + "pk": record.PublicKeyValidator{}, + }), ds) + + // Publish expired. + eol := time.Now().Add(time.Hour * -1) + err := publisher.Publish(context.Background(), id.PrivateKey(), pathCat, PublishWithEOL(eol)) + require.NoError(t, err) + + // Publish new. + err = publisher.Publish(context.Background(), id.PrivateKey(), pathDog) + require.NoError(t, err) + + // Expect new. + res, err := resolver.Resolve(context.Background(), name.AsPath()) + require.NoError(t, err) + require.Equal(t, pathDog, res.Path) + }) + + t.Run("Resolve prefers newer record", func(t *testing.T) { + t.Parallel() + + id, name, ds, r := makeResolverDependencies() + resolver := NewIPNSResolver(r) + publisher := NewIPNSPublisher(r, ds) + + // Publish one... + err := publisher.Publish(context.Background(), id.PrivateKey(), pathCat, PublishWithEOL(time.Now().Add(time.Hour*2))) + require.NoError(t, err) + + // Publish two... + err = publisher.Publish(context.Background(), id.PrivateKey(), pathDog, PublishWithEOL(time.Now().Add(time.Hour*5))) + require.NoError(t, err) + + // Should receive newer! + res, err := resolver.Resolve(context.Background(), name.AsPath()) + require.NoError(t, err) + require.Equal(t, pathDog, res.Path) + }) +} diff --git a/namesys/namesys.go b/namesys/namesys.go index 381e5bff1..100eec0ce 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -20,7 +20,7 @@ import ( "time" lru "github.com/hashicorp/golang-lru/v2" - opts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -32,35 +32,38 @@ import ( madns "github.com/multiformats/go-multiaddr-dns" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/multierr" ) -// mpns (a multi-protocol NameSystem) implements generic IPFS naming. +// namesys is a multi-protocol [NameSystem] that implements generic IPFS naming. +// It uses several [Resolver]s: // -// Uses several Resolvers: -// (a) IPFS routing naming: SFS-like PKI names. -// (b) dns domains: resolves using links in DNS TXT records +// 1. IPFS routing naming: SFS-like PKI names. +// 2. dns domains: resolves using links in DNS TXT records // -// It can only publish to: (a) IPFS routing naming. -type mpns struct { +// It can only publish to: 1. IPFS routing naming. +type namesys struct { ds ds.Datastore dnsResolver, ipnsResolver resolver ipnsPublisher Publisher - staticMap map[string]path.Path - cache *lru.Cache[string, any] + staticMap map[string]*cacheEntry + cache *lru.Cache[string, cacheEntry] } -type Option func(*mpns) error +var _ NameSystem = &namesys{} + +type Option func(*namesys) error // WithCache is an option that instructs the name system to use a (LRU) cache of the given size. func WithCache(size int) Option { - return func(ns *mpns) error { + return func(ns *namesys) error { if size <= 0 { return fmt.Errorf("invalid cache size %d; must be > 0", size) } - cache, err := lru.New[string, any](size) + cache, err := lru.New[string, cacheEntry](size) if err != nil { return err } @@ -70,33 +73,34 @@ func WithCache(size int) Option { } } -// WithDNSResolver is an option that supplies a custom DNS resolver to use instead of the system -// default. +// WithDNSResolver is an option that supplies a custom DNS resolver to use instead +// of the system default. func WithDNSResolver(rslv madns.BasicResolver) Option { - return func(ns *mpns) error { + return func(ns *namesys) error { ns.dnsResolver = NewDNSResolver(rslv.LookupTXT) return nil } } -// WithDatastore is an option that supplies a datastore to use instead of an in-memory map datastore. The datastore is used to store published IPNS records and make them available for querying. +// WithDatastore is an option that supplies a datastore to use instead of an in-memory map datastore. +// The datastore is used to store published IPNS Records and make them available for querying. func WithDatastore(ds ds.Datastore) Option { - return func(ns *mpns) error { + return func(ns *namesys) error { ns.ds = ds return nil } } -// NewNameSystem will construct the IPFS naming system based on Routing +// NewNameSystem constructs an IPFS [NameSystem] based on the given [routing.ValueStore]. func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { - var staticMap map[string]path.Path + var staticMap map[string]*cacheEntry // Prewarm namesys cache with static records for deterministic tests and debugging. // Useful for testing things like DNSLink without real DNS lookup. // Example: // IPFS_NS_MAP="dnslink-test.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" if list := os.Getenv("IPFS_NS_MAP"); list != "" { - staticMap = make(map[string]path.Path) + staticMap = make(map[string]*cacheEntry) for _, pair := range strings.Split(list, ",") { mapping := strings.SplitN(pair, ":", 2) key := mapping[0] @@ -104,11 +108,11 @@ func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { if err != nil { return nil, err } - staticMap[key] = value + staticMap[ipns.NamespacePrefix+key] = &cacheEntry{val: value, ttl: 0} } } - ns := &mpns{ + ns := &namesys{ staticMap: staticMap, } @@ -127,149 +131,109 @@ func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { ns.dnsResolver = NewDNSResolver(madns.DefaultResolver.LookupTXT) } - ns.ipnsResolver = NewIpnsResolver(r) - ns.ipnsPublisher = NewIpnsPublisher(r, ns.ds) + ns.ipnsResolver = NewIPNSResolver(r) + ns.ipnsPublisher = NewIPNSPublisher(r, ns.ds) return ns, nil } -// DefaultResolverCacheTTL defines max ttl of a record placed in namesys cache. -const DefaultResolverCacheTTL = time.Minute - // Resolve implements Resolver. -func (ns *mpns) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { - ctx, span := StartSpan(ctx, "MPNS.Resolve", trace.WithAttributes(attribute.String("Name", name))) +func (ns *namesys) Resolve(ctx context.Context, p path.Path, options ...ResolveOption) (ResolveResult, error) { + ctx, span := startSpan(ctx, "namesys.Resolve", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - if strings.HasPrefix(name, "/ipfs/") { - return path.NewPath(name) - } - - if !strings.HasPrefix(name, "/") { - return path.NewPath("/ipfs/" + name) - } - - return resolve(ctx, ns, name, opts.ProcessOpts(options)) + return resolve(ctx, ns, p, ProcessResolveOptions(options)) } -func (ns *mpns) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { - ctx, span := StartSpan(ctx, "MPNS.ResolveAsync", trace.WithAttributes(attribute.String("Name", name))) +func (ns *namesys) ResolveAsync(ctx context.Context, p path.Path, options ...ResolveOption) <-chan ResolveAsyncResult { + ctx, span := startSpan(ctx, "namesys.ResolveAsync", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - if strings.HasPrefix(name, "/ipfs/") { - p, err := path.NewPath(name) - res := make(chan Result, 1) - res <- Result{p, err} - close(res) - return res - } - - if !strings.HasPrefix(name, "/") { - p, err := path.NewPath("/ipfs/" + name) - res := make(chan Result, 1) - res <- Result{p, err} - close(res) - return res - } - - return resolveAsync(ctx, ns, name, opts.ProcessOpts(options)) + return resolveAsync(ctx, ns, p, ProcessResolveOptions(options)) } // resolveOnce implements resolver. -func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - ctx, span := StartSpan(ctx, "MPNS.ResolveOnceAsync") +func (ns *namesys) resolveOnceAsync(ctx context.Context, p path.Path, options ResolveOptions) <-chan ResolveAsyncResult { + ctx, span := startSpan(ctx, "namesys.ResolveOnceAsync", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - out := make(chan onceResult, 1) - - if !strings.HasPrefix(name, ipnsPrefix) { - name = ipnsPrefix + name - } - segments := strings.SplitN(name, "/", 4) - if len(segments) < 3 || segments[0] != "" { - log.Debugf("invalid name syntax for %s", name) - out <- onceResult{err: ErrResolveFailed} + out := make(chan ResolveAsyncResult, 1) + if !p.Mutable() { + out <- ResolveAsyncResult{Path: p} close(out) return out } - key := segments[2] - - // Resolver selection: - // 1. if it is a PeerID/CID/multihash resolve through "ipns". - // 2. if it is a domain name, resolve through "dns" - - var res resolver - ipnsKey, err := peer.Decode(key) - // CIDs in IPNS are expected to have libp2p-key multicodec - // We ease the transition by returning a more meaningful error with a valid CID + segments := p.Segments() + resolvablePath, err := path.NewPathFromSegments(segments[0], segments[1]) if err != nil { - ipnsCid, cidErr := cid.Decode(key) - if cidErr == nil && ipnsCid.Version() == 1 && ipnsCid.Type() != cid.Libp2pKey { - fixedCid := cid.NewCidV1(cid.Libp2pKey, ipnsCid.Hash()).String() - codecErr := fmt.Errorf("peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/%s", fixedCid) - log.Debugf("RoutingResolver: could not convert public key hash %q to peer ID: %s\n", key, codecErr) - out <- onceResult{err: codecErr} - close(out) - return out - } - } - - cacheKey := key - if err == nil { - cacheKey = string(ipnsKey) + out <- ResolveAsyncResult{Err: err} + close(out) + return out } - if p, ok := ns.cacheGet(cacheKey); ok { - var err error - if len(segments) > 3 { - p, err = path.Join(p, segments[3]) - } + if resolvedBase, ttl, lastMod, ok := ns.cacheGet(resolvablePath.String()); ok { + p, err = joinPaths(resolvedBase, p) span.SetAttributes(attribute.Bool("CacheHit", true)) span.RecordError(err) - - out <- onceResult{value: p, err: err} + out <- ResolveAsyncResult{Path: p, TTL: ttl, LastMod: lastMod, Err: err} close(out) return out + } else { + span.SetAttributes(attribute.Bool("CacheHit", false)) } - span.SetAttributes(attribute.Bool("CacheHit", false)) - if err == nil { + // Resolver selection: + // 1. If it is an IPNS Name, resolve through IPNS. + // 2. if it is a domain name, resolve through DNSLink. + + var res resolver + if _, err := ipns.NameFromString(segments[1]); err == nil { res = ns.ipnsResolver - } else if _, ok := dns.IsDomainName(key); ok { + } else if _, ok := dns.IsDomainName(segments[1]); ok { res = ns.dnsResolver } else { - out <- onceResult{err: fmt.Errorf("invalid IPNS root: %q", key)} + // CIDs in IPNS are expected to have libp2p-key multicodec + // We ease the transition by returning a more meaningful error with a valid CID + ipnsCid, cidErr := cid.Decode(segments[1]) + if cidErr == nil && ipnsCid.Version() == 1 && ipnsCid.Type() != cid.Libp2pKey { + fixedCid := cid.NewCidV1(cid.Libp2pKey, ipnsCid.Hash()).String() + codecErr := fmt.Errorf("peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/%s", fixedCid) + log.Debugf("RoutingResolver: could not convert public key hash %q to peer ID: %s\n", segments[1], codecErr) + out <- ResolveAsyncResult{Err: codecErr} + } else { + out <- ResolveAsyncResult{Err: fmt.Errorf("cannot resolve: %q", resolvablePath.String())} + } + close(out) return out } - resCh := res.resolveOnceAsync(ctx, key, options) - var best onceResult + resCh := res.resolveOnceAsync(ctx, resolvablePath, options) + var best ResolveAsyncResult go func() { defer close(out) for { select { case res, ok := <-resCh: if !ok { - if best != (onceResult{}) { - ns.cacheSet(cacheKey, best.value, best.ttl) + if best != (ResolveAsyncResult{}) { + ns.cacheSet(resolvablePath.String(), best.Path, best.TTL, best.LastMod) } return } - if res.err == nil { + + if res.Err == nil { best = res } - p := res.value - err := res.err - ttl := res.ttl - // Attach rest of the path - if len(segments) > 3 { - p, err = path.Join(p, segments[3]) + p, err := joinPaths(res.Path, p) + if err != nil { + // res.Err may already be defined, so just combine them + res.Err = multierr.Combine(err, res.Err) } - emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl, err: err}) + emitOnceResult(ctx, out, ResolveAsyncResult{Path: p, TTL: res.TTL, LastMod: res.LastMod, Err: res.Err}) case <-ctx.Done(): return } @@ -279,7 +243,7 @@ func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts. return out } -func emitOnceResult(ctx context.Context, outCh chan<- onceResult, r onceResult) { +func emitOnceResult(ctx context.Context, outCh chan<- ResolveAsyncResult, r ResolveAsyncResult) { select { case outCh <- r: case <-ctx.Done(): @@ -287,30 +251,35 @@ func emitOnceResult(ctx context.Context, outCh chan<- onceResult, r onceResult) } // Publish implements Publisher -func (ns *mpns) Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...opts.PublishOption) error { - ctx, span := StartSpan(ctx, "MPNS.Publish") +func (ns *namesys) Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...PublishOption) error { + ctx, span := startSpan(ctx, "MPNS.Publish") defer span.End() // This is a bit hacky. We do this because the EOL is based on the current // time, but also needed in the end of the function. Therefore, we parse // the options immediately and add an option PublishWithEOL with the EOL // calculated in this moment. - publishOpts := opts.ProcessPublishOptions(options) - options = append(options, opts.PublishWithEOL(publishOpts.EOL)) + publishOpts := ProcessPublishOptions(options) + options = append(options, PublishWithEOL(publishOpts.EOL)) - id, err := peer.IDFromPrivateKey(name) + pid, err := peer.IDFromPrivateKey(name) if err != nil { span.RecordError(err) return err } - span.SetAttributes(attribute.String("ID", id.String())) + + ipnsName := ipns.NameFromPeer(pid) + cacheKey := ipnsName.String() + + span.SetAttributes(attribute.String("ID", pid.String())) if err := ns.ipnsPublisher.Publish(ctx, name, value, options...); err != nil { // Invalidate the cache. Publishing may _partially_ succeed but // still return an error. - ns.cacheInvalidate(string(id)) + ns.cacheInvalidate(cacheKey) span.RecordError(err) return err } + ttl := DefaultResolverCacheTTL if publishOpts.TTL >= 0 { ttl = publishOpts.TTL @@ -318,6 +287,20 @@ func (ns *mpns) Publish(ctx context.Context, name ci.PrivKey, value path.Path, o if ttEOL := time.Until(publishOpts.EOL); ttEOL < ttl { ttl = ttEOL } - ns.cacheSet(string(id), value, ttl) + ns.cacheSet(cacheKey, value, ttl, time.Now()) return nil } + +// Resolve is an utility function that takes a [NameSystem] and a [path.Path], and +// returns the result of [NameSystem.Resolve] for the given path. If the given namesys +// is nil, [ErrNoNamesys] is returned. +func Resolve(ctx context.Context, ns NameSystem, p path.Path) (ResolveResult, error) { + ctx, span := startSpan(ctx, "Resolve", trace.WithAttributes(attribute.Stringer("Path", p))) + defer span.End() + + if ns == nil { + return ResolveResult{}, ErrNoNamesys + } + + return ns.Resolve(ctx, p) +} diff --git a/namesys/namesys_cache.go b/namesys/namesys_cache.go new file mode 100644 index 000000000..fc8842e3b --- /dev/null +++ b/namesys/namesys_cache.go @@ -0,0 +1,78 @@ +package namesys + +import ( + "time" + + "github.com/ipfs/boxo/path" +) + +type cacheEntry struct { + val path.Path // is the value of this entry + ttl time.Duration // is the ttl of this entry + lastMod time.Time // is the last time this entry was modified + cacheEOL time.Time // is until when we keep this entry in cache +} + +func (ns *namesys) cacheGet(name string) (path.Path, time.Duration, time.Time, bool) { + // existence of optional mapping defined via IPFS_NS_MAP is checked first + if ns.staticMap != nil { + entry, ok := ns.staticMap[name] + if ok { + return entry.val, entry.ttl, entry.lastMod, true + } + } + + if ns.cache == nil { + return nil, 0, time.Now(), false + } + + entry, ok := ns.cache.Get(name) + if !ok { + return nil, 0, time.Now(), false + } + + if time.Now().Before(entry.cacheEOL) { + return entry.val, entry.ttl, entry.lastMod, true + } + + // We do not delete the entry from the cache. Removals are handled by the + // backing cache system. It is useful to keep it since cacheSet can use + // previously existing values to heuristically update a cache entry. + return nil, 0, time.Now(), false +} + +func (ns *namesys) cacheSet(name string, val path.Path, ttl time.Duration, lastMod time.Time) { + if ns.cache == nil || ttl <= 0 { + return + } + + // Set the current date if there's no lastMod. + if lastMod.IsZero() { + lastMod = time.Now() + } + + // If there's an already cached version with the same path, but + // different lastMod date, keep the oldest. + entry, ok := ns.cache.Get(name) + if ok && entry.val.String() == val.String() { + if lastMod.After(entry.lastMod) { + lastMod = entry.lastMod + } + } + + // Add automatically evicts previous entry, so it works for updating. + ns.cache.Add(name, cacheEntry{ + val: val, + ttl: ttl, + lastMod: lastMod, + cacheEOL: time.Now().Add(ttl), + }) +} + +func (ns *namesys) cacheInvalidate(name string) { + if ns.cache == nil { + return + } + + ns.cache.Remove(name) +} diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go index 9e1d9f5f6..89c4ba3d4 100644 --- a/namesys/namesys_test.go +++ b/namesys/namesys_test.go @@ -2,12 +2,9 @@ package namesys import ( "context" - "errors" - "fmt" "testing" "time" - opts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" offroute "github.com/ipfs/boxo/routing/offline" @@ -16,33 +13,33 @@ import ( record "github.com/libp2p/go-libp2p-record" ci "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type mockResolver struct { entries map[string]string } -func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expError error) { +func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expectedTTL time.Duration, expectedError error) { t.Helper() - p, err := resolver.Resolve(context.Background(), name, opts.Depth(depth)) - if !errors.Is(err, expError) { - t.Fatal(fmt.Errorf( - "expected %s with a depth of %d to have a '%s' error, but got '%s'", - name, depth, expError, err)) - } + + ptr, err := path.NewPath(name) + require.NoError(t, err) + + res, err := resolver.Resolve(context.Background(), ptr, ResolveWithDepth(depth)) + require.ErrorIs(t, err, expectedError) + require.Equal(t, expectedTTL, res.TTL) if expected == "" { - assert.Nil(t, p, "%s with depth %d", name, depth) + require.Nil(t, res.Path, "%s with depth %d", name, depth) } else { - assert.Equal(t, p.String(), expected, "%s with depth %d", name, depth) + require.Equal(t, expected, res.Path.String(), "%s with depth %d", name, depth) } } -func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - p, err := path.NewPath(r.entries[name]) - out := make(chan onceResult, 1) - out <- onceResult{value: p, err: err} +func (r *mockResolver) resolveOnceAsync(ctx context.Context, p path.Path, options ResolveOptions) <-chan ResolveAsyncResult { + p, err := path.NewPath(r.entries[p.String()]) + out := make(chan ResolveAsyncResult, 1) + out <- ResolveAsyncResult{Path: p, Err: err} close(out) return out } @@ -50,12 +47,12 @@ func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, option func mockResolverOne() *mockResolver { return &mockResolver{ entries: map[string]string{ - "QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy": "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", - "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n": "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", - "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD": "/ipns/ipfs.io", - "QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei": "/ipfs/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act", - "12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // ed25519+identity multihash - "bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // cidv1 in base32 with libp2p-key multicodec + "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy": "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", + "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n": "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", + "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD": "/ipns/ipfs.io", + "/ipns/QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei": "/ipfs/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act", + "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // ed25519+identity multihash + "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // cidv1 in base32 with libp2p-key multicodec }, } } @@ -63,123 +60,106 @@ func mockResolverOne() *mockResolver { func mockResolverTwo() *mockResolver { return &mockResolver{ entries: map[string]string{ - "ipfs.io": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", + "/ipns/ipfs.io": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", }, } } func TestNamesysResolution(t *testing.T) { - r := &mpns{ + r := &namesys{ ipnsResolver: mockResolverOne(), dnsResolver: mockResolverTwo(), } - testResolution(t, r, "Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/ipfs.io", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/ipfs.io", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/ipfs.io", 2, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) + for _, testCase := range []struct { + name string + depth uint + expectedPath string + expectedTTL time.Duration + expectedError error + }{ + {"/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion}, + {"/ipns/ipfs.io", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/ipfs.io", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion}, + {"/ipns/ipfs.io", 2, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion}, + {"/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", 0, ErrResolveRecursion}, + {"/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion}, + {"/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion}, + {"/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion}, + {"/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion}, + } { + t.Run(testCase.name, func(t *testing.T) { + testResolution(t, r, testCase.name, (testCase.depth), testCase.expectedPath, 0, testCase.expectedError) + }) + } +} + +func TestResolveIPNS(t *testing.T) { + ns := &namesys{ + ipnsResolver: mockResolverOne(), + dnsResolver: mockResolverTwo(), + } + + inputPath, err := path.NewPath("/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy/a/b/c") + require.NoError(t, err) + + res, err := Resolve(context.Background(), ns, inputPath) + require.NoError(t, err) + require.Equal(t, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj/a/b/c", res.Path.String()) } func TestPublishWithCache0(t *testing.T) { dst := dssync.MutexWrap(ds.NewMapDatastore()) - priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048) - if err != nil { - t.Fatal(err) - } - ps, err := pstoremem.NewPeerstore() - if err != nil { - t.Fatal(err) - } - pid, err := peer.IDFromPrivateKey(priv) - if err != nil { - t.Fatal(err) - } - err = ps.AddPrivKey(pid, priv) - if err != nil { - t.Fatal(err) - } + priv, _, err := ci.GenerateKeyPair(ci.RSA, 4096) + require.NoError(t, err) routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{ - "ipns": ipns.Validator{KeyBook: ps}, + "ipns": ipns.Validator{}, // No need for KeyBook, as records created by NameSys include PublicKey for RSA. "pk": record.PublicKeyValidator{}, }) nsys, err := NewNameSystem(routing, WithDatastore(dst)) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // CID is arbitrary. p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + err = nsys.Publish(context.Background(), priv, p) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) } func TestPublishWithTTL(t *testing.T) { dst := dssync.MutexWrap(ds.NewMapDatastore()) priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048) - if err != nil { - t.Fatal(err) - } - ps, err := pstoremem.NewPeerstore() - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + pid, err := peer.IDFromPrivateKey(priv) - if err != nil { - t.Fatal(err) - } - err = ps.AddPrivKey(pid, priv) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{ - "ipns": ipns.Validator{KeyBook: ps}, + "ipns": ipns.Validator{}, // No need for KeyBook, as records created by NameSys include PublicKey for RSA. "pk": record.PublicKeyValidator{}, }) - nsys, err := NewNameSystem(routing, WithDatastore(dst), WithCache(128)) - if err != nil { - t.Fatal(err) - } + ns, err := NewNameSystem(routing, WithDatastore(dst), WithCache(128)) + require.NoError(t, err) // CID is arbitrary. p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ttl := 1 * time.Second eol := time.Now().Add(2 * time.Second) - err = nsys.Publish(context.Background(), priv, p, opts.PublishWithEOL(eol), opts.PublishWithTTL(ttl)) - if err != nil { - t.Fatal(err) - } - ientry, ok := nsys.(*mpns).cache.Get(string(pid)) - if !ok { - t.Fatal("cache get failed") - } - entry, ok := ientry.(cacheEntry) - if !ok { - t.Fatal("bad cache item returned") - } - if entry.eol.Sub(eol) > 10*time.Millisecond { - t.Fatalf("bad cache ttl: expected %s, got %s", eol, entry.eol) - } + err = ns.Publish(context.Background(), priv, p, PublishWithEOL(eol), PublishWithTTL(ttl)) + require.NoError(t, err) + + entry, ok := ns.(*namesys).cache.Get(ipns.NameFromPeer(pid).String()) + require.True(t, ok) + require.LessOrEqual(t, entry.cacheEOL.Sub(eol), 10*time.Millisecond) } diff --git a/namesys/publisher_test.go b/namesys/publisher_test.go deleted file mode 100644 index 536e72771..000000000 --- a/namesys/publisher_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package namesys - -import ( - "context" - "crypto/rand" - "testing" - "time" - - "github.com/ipfs/boxo/path" - - dshelp "github.com/ipfs/boxo/datastore/dshelp" - "github.com/ipfs/boxo/ipns" - mockrouting "github.com/ipfs/boxo/routing/mock" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - testutil "github.com/libp2p/go-libp2p-testing/net" - ci "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/peer" - ma "github.com/multiformats/go-multiaddr" -) - -type identity struct { - testutil.PeerNetParams -} - -func (p *identity) ID() peer.ID { - return p.PeerNetParams.ID -} - -func (p *identity) Address() ma.Multiaddr { - return p.Addr -} - -func (p *identity) PrivateKey() ci.PrivKey { - return p.PrivKey -} - -func (p *identity) PublicKey() ci.PubKey { - return p.PubKey -} - -func testNamekeyPublisher(t *testing.T, keyType int, expectedErr error, expectedExistence bool) { - // Context - ctx := context.Background() - - // Private key - privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader) - if err != nil { - t.Fatal(err) - } - - // ID - id, err := peer.IDFromPublicKey(pubKey) - if err != nil { - t.Fatal(err) - } - - // Value - value, err := path.NewPath("/ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") - if err != nil { - t.Fatal(err) - } - - // Seqnum - seqnum := uint64(0) - - // Eol - eol := time.Now().Add(24 * time.Hour) - - // Routing value store - p := testutil.PeerNetParams{ - ID: id, - PrivKey: privKey, - PubKey: pubKey, - Addr: testutil.ZeroLocalTCPAddress, - } - - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - serv := mockrouting.NewServer() - r := serv.ClientWithDatastore(context.Background(), &identity{p}, dstore) - - rec, err := ipns.NewRecord(privKey, value, seqnum, eol, 0) - if err != nil { - t.Fatal(err) - } - - err = PutRecordToRouting(ctx, r, pubKey, rec) - if err != nil { - t.Fatal(err) - } - - // Check for namekey existence in value store - namekey := PkKeyForID(id) - _, err = r.GetValue(ctx, namekey) - if err != expectedErr { - t.Fatal(err) - } - - // Also check datastore for completeness - key := dshelp.NewKeyFromBinary([]byte(namekey)) - exists, err := dstore.Has(ctx, key) - if err != nil { - t.Fatal(err) - } - - if exists != expectedExistence { - t.Fatal("Unexpected key existence in datastore") - } -} - -func TestRSAPublisher(t *testing.T) { - testNamekeyPublisher(t, ci.RSA, nil, true) -} - -func TestEd22519Publisher(t *testing.T) { - testNamekeyPublisher(t, ci.Ed25519, ds.ErrNotFound, false) -} - -func TestAsyncDS(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - rt := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t)) - ds := &checkSyncDS{ - Datastore: ds.NewMapDatastore(), - syncKeys: make(map[ds.Key]struct{}), - } - publisher := NewIpnsPublisher(rt, ds) - - ipnsFakeID := testutil.RandIdentityOrFatal(t) - ipnsVal, err := path.NewPath("/ipns/foo.bar") - if err != nil { - t.Fatal(err) - } - - if err := publisher.Publish(ctx, ipnsFakeID.PrivateKey(), ipnsVal); err != nil { - t.Fatal(err) - } - - ipnsKey := IpnsDsKey(ipnsFakeID.ID()) - - for k := range ds.syncKeys { - if k.IsAncestorOf(ipnsKey) || k.Equal(ipnsKey) { - return - } - } - - t.Fatal("ipns key not synced") -} - -type checkSyncDS struct { - ds.Datastore - syncKeys map[ds.Key]struct{} -} - -func (d *checkSyncDS) Sync(ctx context.Context, prefix ds.Key) error { - d.syncKeys[prefix] = struct{}{} - return d.Datastore.Sync(ctx, prefix) -} diff --git a/namesys/republisher/repub.go b/namesys/republisher/repub.go index bb7a5e2b0..2f53cade6 100644 --- a/namesys/republisher/repub.go +++ b/namesys/republisher/repub.go @@ -5,13 +5,15 @@ package republisher import ( "context" "errors" + "fmt" "time" - keystore "github.com/ipfs/boxo/keystore" + "github.com/ipfs/boxo/keystore" "github.com/ipfs/boxo/namesys" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" - opts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/ipns" ds "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" @@ -21,24 +23,27 @@ import ( "github.com/libp2p/go-libp2p/core/peer" ) -var errNoEntry = errors.New("no previous entry") - -var log = logging.Logger("ipns-repub") +var ( + errNoEntry = errors.New("no previous entry") + log = logging.Logger("ipns/repub") +) -// DefaultRebroadcastInterval is the default interval at which we rebroadcast IPNS records -var DefaultRebroadcastInterval = time.Hour * 4 +const ( + // DefaultRebroadcastInterval is the default interval at which we rebroadcast IPNS records + DefaultRebroadcastInterval = time.Hour * 4 -// InitialRebroadcastDelay is the delay before first broadcasting IPNS records on start -var InitialRebroadcastDelay = time.Minute * 1 + // InitialRebroadcastDelay is the delay before first broadcasting IPNS records on start + InitialRebroadcastDelay = time.Minute * 1 -// FailureRetryInterval is the interval at which we retry IPNS records broadcasts (when they fail) -var FailureRetryInterval = time.Minute * 5 + // FailureRetryInterval is the interval at which we retry IPNS records broadcasts (when they fail) + FailureRetryInterval = time.Minute * 5 -// DefaultRecordLifetime is the default lifetime for IPNS records -const DefaultRecordLifetime = time.Hour * 24 + // DefaultRecordLifetime is the default lifetime for IPNS records + DefaultRecordLifetime = time.Hour * 24 +) // Republisher facilitates the regular publishing of all the IPNS records -// associated to keys in a Keystore. +// associated to keys in a [keystore.Keystore]. type Republisher struct { ns namesys.Publisher ds ds.Datastore @@ -51,7 +56,7 @@ type Republisher struct { RecordLifetime time.Duration } -// NewRepublisher creates a new Republisher +// NewRepublisher creates a new [Republisher] from the given options. func NewRepublisher(ns namesys.Publisher, ds ds.Datastore, self ic.PrivKey, ks keystore.Keystore) *Republisher { return &Republisher{ ns: ns, @@ -63,8 +68,7 @@ func NewRepublisher(ns namesys.Publisher, ds ds.Datastore, self ic.PrivKey, ks k } } -// Run starts the republisher facility. It can be stopped by stopping the -// provided proc. +// Run starts the republisher facility. It can be stopped by stopping the provided proc. func (rp *Republisher) Run(proc goprocess.Process) { timer := time.NewTimer(InitialRebroadcastDelay) defer timer.Stop() @@ -92,7 +96,7 @@ func (rp *Republisher) Run(proc goprocess.Process) { func (rp *Republisher) republishEntries(p goprocess.Process) error { ctx, cancel := context.WithCancel(gpctx.OnClosingContext(p)) defer cancel() - ctx, span := namesys.StartSpan(ctx, "Republisher.RepublishEntries") + ctx, span := startSpan(ctx, "Republisher.RepublishEntries") defer span.End() // TODO: Use rp.ipns.ListPublished(). We can't currently *do* that @@ -126,7 +130,7 @@ func (rp *Republisher) republishEntries(p goprocess.Process) error { } func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) error { - ctx, span := namesys.StartSpan(ctx, "Republisher.RepublishEntry") + ctx, span := startSpan(ctx, "Republisher.RepublishEntry") defer span.End() id, err := peer.IDFromPrivateKey(priv) if err != nil { @@ -164,7 +168,7 @@ func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) erro if prevEol.After(eol) { eol = prevEol } - err = rp.ns.Publish(ctx, priv, p, opts.PublishWithEOL(eol)) + err = rp.ns.Publish(ctx, priv, p, namesys.PublishWithEOL(eol)) span.RecordError(err) return err } @@ -182,3 +186,9 @@ func (rp *Republisher) getLastIPNSRecord(ctx context.Context, id peer.ID) (*ipns return ipns.UnmarshalRecord(val) } + +var tracer = otel.Tracer("boxo/namesys/republisher") + +func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return tracer.Start(ctx, fmt.Sprintf("Namesys.%s", name)) +} diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go index 6b5d2abf0..2759e21b7 100644 --- a/namesys/republisher/repub_test.go +++ b/namesys/republisher/repub_test.go @@ -13,8 +13,8 @@ import ( host "github.com/libp2p/go-libp2p/core/host" peer "github.com/libp2p/go-libp2p/core/peer" routing "github.com/libp2p/go-libp2p/core/routing" + "github.com/stretchr/testify/require" - opts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" ds "github.com/ipfs/go-datastore" @@ -27,7 +27,7 @@ import ( type mockNode struct { h host.Host - id string + id peer.ID privKey ic.PrivKey store ds.Batching dht *dht.IpfsDHT @@ -47,13 +47,11 @@ func getMockNode(t *testing.T, ctx context.Context) *mockNode { return rt, err }), ) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) return &mockNode{ h: h, - id: h.ID().Pretty(), + id: h.ID(), privKey: h.Peerstore().PrivKey(h.ID()), store: dstore, dht: idht, @@ -72,9 +70,7 @@ func TestRepublish(t *testing.T) { for i := 0; i < 10; i++ { n := getMockNode(t, ctx) ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store)) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) nsystems = append(nsystems, ns) nodes = append(nodes, n) @@ -83,21 +79,18 @@ func TestRepublish(t *testing.T) { pinfo := host.InfoFromHost(nodes[0].h) for _, n := range nodes[1:] { - if err := n.h.Connect(ctx, *pinfo); err != nil { - t.Fatal(err) - } + err := n.h.Connect(ctx, *pinfo) + require.NoError(t, err) } // have one node publish a record that is valid for 1 second publisher := nodes[3] p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) - rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store) - name := "/ipns/" + publisher.id + rp := namesys.NewIPNSPublisher(publisher.dht, publisher.store) + name := ipns.NameFromPeer(publisher.id).AsPath() // Retry in case the record expires before we can fetch it. This can // happen when running the test on a slow machine. @@ -105,10 +98,8 @@ func TestRepublish(t *testing.T) { timeout := time.Second for { expiration = time.Now().Add(time.Second) - err := rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration)) - if err != nil { - t.Fatal(err) - } + err := rp.Publish(ctx, publisher.privKey, p, namesys.PublishWithEOL(expiration)) + require.NoError(t, err) err = verifyResolution(nsystems, name, p) if err == nil { @@ -124,9 +115,8 @@ func TestRepublish(t *testing.T) { // Now wait a second, the records will be invalid and we should fail to resolve time.Sleep(timeout) - if err := verifyResolutionFails(nsystems, name); err != nil { - t.Fatal(err) - } + err = verifyResolutionFails(nsystems, name) + require.NoError(t, err) // The republishers that are contained within the nodes have their timeout set // to 12 hours. Instead of trying to tweak those, we're just going to pretend @@ -142,9 +132,8 @@ func TestRepublish(t *testing.T) { time.Sleep(time.Second * 2) // we should be able to resolve them now - if err := verifyResolution(nsystems, name, p); err != nil { - t.Fatal(err) - } + err = verifyResolution(nsystems, name, p) + require.NoError(t, err) } func TestLongEOLRepublish(t *testing.T) { @@ -158,9 +147,7 @@ func TestLongEOLRepublish(t *testing.T) { for i := 0; i < 10; i++ { n := getMockNode(t, ctx) ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store)) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) nsystems = append(nsystems, ns) nodes = append(nodes, n) @@ -169,31 +156,24 @@ func TestLongEOLRepublish(t *testing.T) { pinfo := host.InfoFromHost(nodes[0].h) for _, n := range nodes[1:] { - if err := n.h.Connect(ctx, *pinfo); err != nil { - t.Fatal(err) - } + err := n.h.Connect(ctx, *pinfo) + require.NoError(t, err) } // have one node publish a record that is valid for 1 second publisher := nodes[3] - p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid - if err != nil { - t.Fatal(err) - } + p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + require.NoError(t, err) - rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store) - name := "/ipns/" + publisher.id + rp := namesys.NewIPNSPublisher(publisher.dht, publisher.store) + name := ipns.NameFromPeer(publisher.id).AsPath() expiration := time.Now().Add(time.Hour) - err = rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration)) - if err != nil { - t.Fatal(err) - } + err = rp.Publish(ctx, publisher.privKey, p, namesys.PublishWithEOL(expiration)) + require.NoError(t, err) err = verifyResolution(nsystems, name, p) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // The republishers that are contained within the nodes have their timeout set // to 12 hours. Instead of trying to tweak those, we're just going to pretend @@ -209,23 +189,14 @@ func TestLongEOLRepublish(t *testing.T) { time.Sleep(time.Second * 2) err = verifyResolution(nsystems, name, p) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) rec, err := getLastIPNSRecord(ctx, publisher.store, publisher.h.ID()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) finalEol, err := rec.Validity() - if err != nil { - t.Fatal(err) - } - - if !finalEol.Equal(expiration) { - t.Fatal("expiration time modified") - } + require.NoError(t, err) + require.Equal(t, expiration.UTC(), finalEol.UTC()) } func getLastIPNSRecord(ctx context.Context, dstore ds.Datastore, id peer.ID) (*ipns.Record, error) { @@ -238,23 +209,23 @@ func getLastIPNSRecord(ctx context.Context, dstore ds.Datastore, id peer.ID) (*i return ipns.UnmarshalRecord(val) } -func verifyResolution(nsystems []namesys.NameSystem, key string, exp path.Path) error { +func verifyResolution(nsystems []namesys.NameSystem, key path.Path, exp path.Path) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() for _, n := range nsystems { - val, err := n.Resolve(ctx, key) + res, err := n.Resolve(ctx, key) if err != nil { return err } - if val.String() != exp.String() { + if res.Path.String() != exp.String() { return errors.New("resolved wrong record") } } return nil } -func verifyResolutionFails(nsystems []namesys.NameSystem, key string) error { +func verifyResolutionFails(nsystems []namesys.NameSystem, key path.Path) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() for _, n := range nsystems { diff --git a/namesys/resolve/resolve.go b/namesys/resolve/resolve.go deleted file mode 100644 index b01fd38f7..000000000 --- a/namesys/resolve/resolve.go +++ /dev/null @@ -1,56 +0,0 @@ -package resolve - -import ( - "context" - "errors" - "fmt" - - "github.com/ipfs/boxo/path" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/ipfs/boxo/namesys" -) - -// ErrNoNamesys is an explicit error for when an IPFS node doesn't -// (yet) have a name system -var ErrNoNamesys = errors.New( - "core/resolve: no Namesys on IpfsNode - can't resolve ipns entry") - -// ResolveIPNS resolves /ipns paths -func ResolveIPNS(ctx context.Context, nsys namesys.NameSystem, p path.Path) (path.Path, error) { - ctx, span := namesys.StartSpan(ctx, "ResolveIPNS", trace.WithAttributes(attribute.String("Path", p.String()))) - defer span.End() - - if p.Namespace() == path.IPNSNamespace { - // TODO(cryptix): we should be able to query the local cache for the path - if nsys == nil { - return nil, ErrNoNamesys - } - - seg := p.Segments() - - if len(seg) < 2 || seg[1] == "" { // just "/" without further segments - err := fmt.Errorf("invalid path %q: ipns path missing IPNS ID", p) - return nil, err - } - - extensions := seg[2:] - resolvable, err := path.NewPathFromSegments(seg[0], seg[1]) - if err != nil { - return nil, err - } - - respath, err := nsys.Resolve(ctx, resolvable.String()) - if err != nil { - return nil, err - } - - segments := append(respath.Segments(), extensions...) - p, err = path.NewPathFromSegments(segments...) - if err != nil { - return nil, err - } - } - return p, nil -} diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go deleted file mode 100644 index 158a9a26c..000000000 --- a/namesys/resolve_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package namesys - -import ( - "context" - "errors" - "testing" - "time" - - ipns "github.com/ipfs/boxo/ipns" - "github.com/ipfs/boxo/path" - mockrouting "github.com/ipfs/boxo/routing/mock" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - tnet "github.com/libp2p/go-libp2p-testing/net" -) - -func TestRoutingResolve(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - serv := mockrouting.NewServer() - id := tnet.RandIdentityOrFatal(t) - d := serv.ClientWithDatastore(context.Background(), id, dstore) - - resolver := NewIpnsResolver(d) - publisher := NewIpnsPublisher(d, dstore) - - identity := tnet.RandIdentityOrFatal(t) - - h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") - if err != nil { - t.Fatal(err) - } - - err = publisher.Publish(context.Background(), identity.PrivateKey(), h) - if err != nil { - t.Fatal(err) - } - - res, err := resolver.Resolve(context.Background(), identity.ID().Pretty()) - if err != nil { - t.Fatal(err) - } - - if res.String() != h.String() { - t.Fatal("Got back incorrect value.") - } -} - -func TestPrexistingExpiredRecord(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - d := mockrouting.NewServer().ClientWithDatastore(context.Background(), tnet.RandIdentityOrFatal(t), dstore) - - resolver := NewIpnsResolver(d) - publisher := NewIpnsPublisher(d, dstore) - - identity := tnet.RandIdentityOrFatal(t) - - // Make an expired record and put it in the datastore - h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") - if err != nil { - t.Fatal(err) - } - eol := time.Now().Add(time.Hour * -1) - - entry, err := ipns.NewRecord(identity.PrivateKey(), h, 0, eol, 0) - if err != nil { - t.Fatal(err) - } - err = PutRecordToRouting(context.Background(), d, identity.PublicKey(), entry) - if err != nil { - t.Fatal(err) - } - - // Now, with an old record in the system already, try and publish a new one - err = publisher.Publish(context.Background(), identity.PrivateKey(), h) - if err != nil { - t.Fatal(err) - } - - err = verifyCanResolve(resolver, identity.ID().Pretty(), h) - if err != nil { - t.Fatal(err) - } -} - -func TestPrexistingRecord(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - d := mockrouting.NewServer().ClientWithDatastore(context.Background(), tnet.RandIdentityOrFatal(t), dstore) - - resolver := NewIpnsResolver(d) - publisher := NewIpnsPublisher(d, dstore) - - identity := tnet.RandIdentityOrFatal(t) - - // Make a good record and put it in the datastore - h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") - if err != nil { - t.Fatal(err) - } - eol := time.Now().Add(time.Hour) - entry, err := ipns.NewRecord(identity.PrivateKey(), h, 0, eol, 0) - if err != nil { - t.Fatal(err) - } - err = PutRecordToRouting(context.Background(), d, identity.PublicKey(), entry) - if err != nil { - t.Fatal(err) - } - - // Now, with an old record in the system already, try and publish a new one - err = publisher.Publish(context.Background(), identity.PrivateKey(), h) - if err != nil { - t.Fatal(err) - } - - err = verifyCanResolve(resolver, identity.ID().Pretty(), h) - if err != nil { - t.Fatal(err) - } -} - -func verifyCanResolve(r Resolver, name string, exp path.Path) error { - res, err := r.Resolve(context.Background(), name) - if err != nil { - return err - } - - if res.String() != exp.String() { - return errors.New("got back wrong record") - } - - return nil -} diff --git a/namesys/routing.go b/namesys/routing.go deleted file mode 100644 index 1153341ab..000000000 --- a/namesys/routing.go +++ /dev/null @@ -1,147 +0,0 @@ -package namesys - -import ( - "context" - "strings" - "time" - - opts "github.com/ipfs/boxo/coreiface/options/namesys" - "github.com/ipfs/boxo/ipns" - "github.com/ipfs/boxo/path" - logging "github.com/ipfs/go-log/v2" - dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -var log = logging.Logger("namesys") - -// IpnsResolver implements NSResolver for the main IPFS SFS-like naming -type IpnsResolver struct { - routing routing.ValueStore -} - -// NewIpnsResolver constructs a name resolver using the IPFS Routing system -// to implement SFS-like naming on top. -func NewIpnsResolver(route routing.ValueStore) *IpnsResolver { - if route == nil { - panic("attempt to create resolver with nil routing system") - } - return &IpnsResolver{ - routing: route, - } -} - -// Resolve implements Resolver. -func (r *IpnsResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { - ctx, span := StartSpan(ctx, "IpnsResolver.Resolve", trace.WithAttributes(attribute.String("Name", name))) - defer span.End() - return resolve(ctx, r, name, opts.ProcessOpts(options)) -} - -// ResolveAsync implements Resolver. -func (r *IpnsResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { - ctx, span := StartSpan(ctx, "IpnsResolver.ResolveAsync", trace.WithAttributes(attribute.String("Name", name))) - defer span.End() - return resolveAsync(ctx, r, name, opts.ProcessOpts(options)) -} - -// resolveOnce implements resolver. Uses the IPFS routing system to -// resolve SFS-like names. -func (r *IpnsResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - ctx, span := StartSpan(ctx, "IpnsResolver.ResolveOnceAsync", trace.WithAttributes(attribute.String("Name", name))) - defer span.End() - - out := make(chan onceResult, 1) - log.Debugf("RoutingResolver resolving %s", name) - cancel := func() {} - - if options.DhtTimeout != 0 { - // Resolution must complete within the timeout - ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout) - } - - name = strings.TrimPrefix(name, "/ipns/") - - pid, err := peer.Decode(name) - if err != nil { - log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err) - out <- onceResult{err: err} - close(out) - cancel() - return out - } - - // Use the routing system to get the name. - // Note that the DHT will call the ipns validator when retrieving - // the value, which in turn verifies the ipns record signature - ipnsKey := string(ipns.NameFromPeer(pid).RoutingKey()) - - vals, err := r.routing.SearchValue(ctx, ipnsKey, dht.Quorum(int(options.DhtRecordCount))) - if err != nil { - log.Debugf("RoutingResolver: dht get for name %s failed: %s", name, err) - out <- onceResult{err: err} - close(out) - cancel() - return out - } - - go func() { - defer cancel() - defer close(out) - ctx, span := StartSpan(ctx, "IpnsResolver.ResolveOnceAsync.Worker") - defer span.End() - - for { - select { - case val, ok := <-vals: - if !ok { - return - } - - rec, err := ipns.UnmarshalRecord(val) - if err != nil { - log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", name, err) - emitOnceResult(ctx, out, onceResult{err: err}) - return - } - - p, err := rec.Value() - if err != nil { - emitOnceResult(ctx, out, onceResult{err: err}) - return - } - - ttl := DefaultResolverCacheTTL - if recordTTL, err := rec.TTL(); err == nil { - ttl = recordTTL - } - - switch eol, err := rec.Validity(); err { - case ipns.ErrUnrecognizedValidity: - // No EOL. - case nil: - ttEol := time.Until(eol) - if ttEol < 0 { - // It *was* valid when we first resolved it. - ttl = 0 - } else if ttEol < ttl { - ttl = ttEol - } - default: - log.Errorf("encountered error when parsing EOL: %s", err) - emitOnceResult(ctx, out, onceResult{err: err}) - return - } - - emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl}) - case <-ctx.Done(): - return - } - } - }() - - return out -} diff --git a/namesys/tracing.go b/namesys/tracing.go deleted file mode 100644 index 4ef84294a..000000000 --- a/namesys/tracing.go +++ /dev/null @@ -1,13 +0,0 @@ -package namesys - -import ( - "context" - "fmt" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" -) - -func StartSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { - return otel.Tracer("go-namesys").Start(ctx, fmt.Sprintf("Namesys.%s", name)) -} diff --git a/namesys/utilities.go b/namesys/utilities.go new file mode 100644 index 000000000..2cf4a3871 --- /dev/null +++ b/namesys/utilities.go @@ -0,0 +1,146 @@ +package namesys + +import ( + "context" + "fmt" + "strings" + + "github.com/ipfs/boxo/path" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +type resolver interface { + resolveOnceAsync(context.Context, path.Path, ResolveOptions) <-chan ResolveAsyncResult +} + +// resolve is a helper for implementing Resolver.ResolveN using resolveOnce. +func resolve(ctx context.Context, r resolver, p path.Path, options ResolveOptions) (result ResolveResult, err error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + err = ErrResolveFailed + resCh := resolveAsync(ctx, r, p, options) + + for res := range resCh { + result.Path, result.TTL, result.LastMod, err = res.Path, res.TTL, res.LastMod, res.Err + if err != nil { + break + } + } + + return result, err +} + +func resolveAsync(ctx context.Context, r resolver, p path.Path, options ResolveOptions) <-chan ResolveAsyncResult { + ctx, span := startSpan(ctx, "ResolveAsync") + defer span.End() + + resCh := r.resolveOnceAsync(ctx, p, options) + depth := options.Depth + outCh := make(chan ResolveAsyncResult, 1) + + go func() { + defer close(outCh) + ctx, span := startSpan(ctx, "ResolveAsync.Worker") + defer span.End() + + var subCh <-chan ResolveAsyncResult + var cancelSub context.CancelFunc + defer func() { + if cancelSub != nil { + cancelSub() + } + }() + + for { + select { + case res, ok := <-resCh: + if !ok { + resCh = nil + break + } + + if res.Err != nil { + emitResult(ctx, outCh, res) + return + } + + log.Debugf("resolved %s to %s", p.String(), res.Path.String()) + + if !res.Path.Mutable() { + emitResult(ctx, outCh, res) + break + } + + if depth == 1 { + res.Err = ErrResolveRecursion + emitResult(ctx, outCh, res) + break + } + + subOpts := options + if subOpts.Depth > 1 { + subOpts.Depth-- + } + + var subCtx context.Context + if cancelSub != nil { + // Cancel previous recursive resolve since it won't be used anyways + cancelSub() + } + + subCtx, cancelSub = context.WithCancel(ctx) + _ = cancelSub + + subCh = resolveAsync(subCtx, r, res.Path, subOpts) + case res, ok := <-subCh: + if !ok { + subCh = nil + break + } + + // We don't bother returning here in case of context timeout as there is + // no good reason to do that, and we may still be able to emit a result + emitResult(ctx, outCh, res) + case <-ctx.Done(): + return + } + if resCh == nil && subCh == nil { + return + } + } + }() + return outCh +} + +func emitResult(ctx context.Context, outCh chan<- ResolveAsyncResult, r ResolveAsyncResult) { + select { + case outCh <- r: + case <-ctx.Done(): + } +} + +func joinPaths(resolvedBase, unresolvedPath path.Path) (path.Path, error) { + if resolvedBase == nil { + return nil, nil + } + + segments := unresolvedPath.Segments()[2:] + if strings.HasSuffix(unresolvedPath.String(), "/") { + segments = append(segments, "") + } + + // simple optimization + if len(segments) == 0 { + return resolvedBase, nil + } + + return path.Join(resolvedBase, segments...) +} + +var tracer = otel.Tracer("boxo/namesys") + +func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return tracer.Start(ctx, fmt.Sprintf("Namesys.%s", name)) +}