From f5ac09841b09466fa427cc6187640df94e9cd046 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 22 Mar 2024 19:24:34 +0100 Subject: [PATCH 1/3] feat: support Ipfs-Path-Affinity from IPIP-462 This is first stab at leveraging these hints withing existing boxo/gateway codebase. It is pretty blunt, but will enable smart clients fetching sub-DAGs to work around any content routing gaps For more info and header semantics see https://github.com/ipfs/specs/pull/462 --- gateway/handler.go | 101 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/gateway/handler.go b/gateway/handler.go index 6963bdebf..7460a75ef 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -233,6 +233,8 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { return } + i.handlePathAffinityHints(w, r, contentPath, logger) + // Detect when explicit Accept header or ?format parameter are present responseFormat, formatParams, err := customResponseFormat(r) if err != nil { @@ -752,7 +754,7 @@ func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, } // Detect 'Cache-Control: only-if-cached' in request and return data if it is already in the local datastore. -// https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#cache-control-request-header +// https://specs.ipfs.tech/http-gateways/path-gateway/#cache-control-request-header func (i *handler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath path.Path) bool { if r.Header.Get("Cache-Control") == "only-if-cached" { if !i.backend.IsCached(r.Context(), contentPath) { @@ -887,6 +889,103 @@ func (i *handler) handleSuperfluousNamespace(w http.ResponseWriter, r *http.Requ return true } +// Detect 'Ipfs-Path-Affinity' (IPIP-462) headers in request and use values as a content +// routing hints if passed paths are not already in the local datastore. +// These optional hints are mostly useful for trustless block requests. +// See https://github.com/ipfs/specs/pull/462 +func (i *handler) handlePathAffinityHints(w http.ResponseWriter, r *http.Request, contentPath path.Path, logger *zap.SugaredLogger) { + headerName := "Ipfs-Path-Affinity" + // Skip if no header + if r.Header.Get(headerName) == "" { + return + } + // Skip if contentPath is already locally cached + if i.backend.IsCached(r.Context(), contentPath) { + return + } + // Check canonical header name + // NOTE: we don't use r.Header.Get() because client can send this header more than once + headerValues := r.Header[headerName] + // If not found, try lowercase version. + // NOTE: this is done manually because direct key access does not come with canonicalization, like Header.Get() does + if len(headerValues) == 0 { + headerValues = r.Header[strings.ToLower(headerName)] + } + + // Limit the headerValues to the first 3 items (abuse protection) + if len(headerValues) > 3 { + headerValues = headerValues[:3] + } + + // Process affinity hints + for _, headerValue := range headerValues { + // Non-ascii paths are percent-encoded. + // Decode if the value starts with %2F (percent-encoded '/') + if strings.HasPrefix(headerValue, "%2F") { + decodedValue, err := url.PathUnescape(headerValue) + if err != nil { + logger.Debugw("skipping invalid Ipfs-Path-Affinity hint", "error", err) + continue + } + headerValue = decodedValue + } + // Confirm it is a valid content path + affinityPath, err := path.NewPath(headerValue) + if err != nil { + logger.Debugw("skipping invalid Ipfs-Path-Affinity hint", "error", err) + continue + } + + // Skip duplicated work if immutable affinity hint is a subset of requested immutable contentPath + // (protect against broken clients that use affinity incorrectly) + if !contentPath.Mutable() && !affinityPath.Mutable() && strings.HasPrefix(contentPath.String(), affinityPath.String()) { + logger.Debugw("skipping redundant Ipfs-Path-Affinity hint", "affinity", affinityPath) + continue + } + + // Process hint in background without blocking response logic for contentPath + go func(contentPath path.Path, affinityPath path.Path, logger *zap.SugaredLogger) { + var immutableAffinityPath path.ImmutablePath + logger.Debugw("async processing of Ipfs-Path-Affinity hint", "affinity", affinityPath) + if affinityPath.Mutable() { + // Skip work if mutable affinity hint is a subset of mutable contentPath + if contentPath.Mutable() && strings.HasPrefix(contentPath.String(), affinityPath.String()) { + logger.Debugw("skipping redundant Ipfs-Path-Affinity hint", "affinity", affinityPath) + return + } + immutableAffinityPath, _, _, err = i.backend.ResolveMutable(r.Context(), affinityPath) + if err != nil { + logger.Debugw("error while resolving mutable Ipfs-Path-Affinity hint", "affinity", affinityPath, "error", err) + return + } + } else { + ipath, ok := affinityPath.(path.ImmutablePath) + if !ok { + return + } + immutableAffinityPath = ipath + } + // Skip if affinity path is already cached + if !i.backend.IsCached(r.Context(), immutableAffinityPath) { + // The intention of below code is to asynchronously preconnect + // gateway with providers of the affinityPath in + // Ipfs-Path-Affinity hint. Once connected, these peers can be + // asked directly (via mechanism like bitswap) for blocks + // related to main request for contentPath, and retrieve them, + // even when no other routing system had them announced. If + // original contentPath was received and returned to HTTP + // client before below get is done, the work is cancelled. + + logger.Debugw("started async search for providers of Ipfs-Path-Affinity hint", "affinity", affinityPath) + _, _, err = i.backend.GetBlock(r.Context(), immutableAffinityPath) + logger.Debugw("ended async search for providers of Ipfs-Path-Affinity hint", "affinity", affinityPath, "error", err) + } else { + logger.Debugw("skipping Ipfs-Path-Affinity hint due to data being locally cached", "affinity", affinityPath) + } + }(contentPath, affinityPath, logger) + } +} + // getTemplateGlobalData returns the global data necessary by most templates. func (i *handler) getTemplateGlobalData(r *http.Request, contentPath path.Path) assets.GlobalData { // gatewayURL is used to link to other root CIDs. THis will be blank unless From 6d16b6927f15896bf44c280181df3a19a0fbfaac Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 22 Mar 2024 19:30:23 +0100 Subject: [PATCH 2/3] chore: changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a76d9aab4..e59efb2ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The following emojis are used to highlight certain changes: ### Added * `routing/http/server` now adds `Cache-Control` HTTP header to GET requests: 15 seconds for empty responses, or 5 minutes for responses with providers. +* `gateway` now supports optional `Ipfs-Path-Affinity` hints from [IPIP-462](https://github.com/ipfs/specs/pull/462). ### Changed From 39a8bd72fa8830d97b952178b1b4eb98d71967b9 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 25 Mar 2024 13:10:13 +0100 Subject: [PATCH 3/3] ci: update to go 1.22 on gateway sharness tests --- .github/workflows/gateway-sharness.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gateway-sharness.yml b/.github/workflows/gateway-sharness.yml index 25daf6141..2bd38cb05 100644 --- a/.github/workflows/gateway-sharness.yml +++ b/.github/workflows/gateway-sharness.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: 1.21.x + go-version: 1.22.x - name: Checkout boxo uses: actions/checkout@v3 with: