From ac8f9b1c23c4d4feb8ae97cc0df125d63cf34053 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 6 Oct 2022 15:14:38 +0200 Subject: [PATCH 01/32] wip: play with dag-cbor and dag-json --- core/corehttp/gateway_handler.go | 13 ++++++++ core/corehttp/gateway_handler_json.go | 44 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 core/corehttp/gateway_handler_json.go diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index a96799f5875..3649609e1f1 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -26,6 +26,7 @@ import ( coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" routing "github.com/libp2p/go-libp2p/core/routing" + mc "github.com/multiformats/go-multicodec" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -430,6 +431,14 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request carVersion := formatParams["version"] i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin) return + case "application/vnd.ipld.dag-json": + logger.Debugw("serving dag-json", "path", contentPath) + i.serveJSON(r.Context(), w, r, resolvedPath, begin, "application/vnd.ipld.dag-json", uint64(mc.DagJson)) + return + case "application/vnd.ipld.dag-cbor": + logger.Debugw("serving dag-cbor", "path", contentPath) + i.serveJSON(r.Context(), w, r, resolvedPath, begin, "application/vnd.ipld.dag-cbor", uint64(mc.DagCbor)) + return default: // catch-all for unsuported application/vnd.* err := fmt.Errorf("unsupported format %q", responseFormat) webError(w, "failed respond with requested content type", err, http.StatusBadRequest) @@ -859,6 +868,10 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string] return "application/vnd.ipld.raw", nil, nil case "car": return "application/vnd.ipld.car", nil, nil + case "dag-json": + return "application/vnd.ipld.dag-json", nil, nil + case "dag-cbor": + return "application/vnd.ipld.dag-cbor", nil, nil } } // Browsers and other user agents will send Accept header with generic types like: diff --git a/core/corehttp/gateway_handler_json.go b/core/corehttp/gateway_handler_json.go new file mode 100644 index 00000000000..dc0959393c8 --- /dev/null +++ b/core/corehttp/gateway_handler_json.go @@ -0,0 +1,44 @@ +package corehttp + +import ( + "context" + "fmt" + "html" + "net/http" + "time" + + ipldlegacy "github.com/ipfs/go-ipld-legacy" + ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" +) + +// serveCAR returns a CAR stream for specific DAG+selector +func (i *gatewayHandler) serveJSON(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, begin time.Time, ctype string, codec uint64) { + // ctx, span := tracing.Span(ctx, "Gateway", "ServeCAR", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) + // defer span.End() + // ctx, cancel := context.WithCancel(ctx) + // defer cancel() + + obj, err := i.api.Dag().Get(r.Context(), resolvedPath.Cid()) + if err != nil { + webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) + return + } + + universal, ok := obj.(ipldlegacy.UniversalNode) + if !ok { + webError(w, "todo", fmt.Errorf("%T is not a valid IPLD node", obj), http.StatusInternalServerError) + return + } + finalNode := universal.(ipld.Node) + + encoder, err := multicodec.LookupEncoder(codec) + if err != nil { + webError(w, "todo", err, http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", ctype) + _ = encoder(finalNode, w) +} From 3dce01259be4af4ef8474e2c57b928bdcbffca9b Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 7 Oct 2022 12:03:01 +0200 Subject: [PATCH 02/32] wip: add application/json and application/cbor --- core/corehttp/gateway_handler.go | 14 +++++-- core/corehttp/gateway_handler_codec.go | 58 ++++++++++++++++++++++++++ core/corehttp/gateway_handler_json.go | 44 ------------------- 3 files changed, 69 insertions(+), 47 deletions(-) create mode 100644 core/corehttp/gateway_handler_codec.go delete mode 100644 core/corehttp/gateway_handler_json.go diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 3649609e1f1..a918129693c 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -432,12 +432,14 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin) return case "application/vnd.ipld.dag-json": + case "application/json": logger.Debugw("serving dag-json", "path", contentPath) - i.serveJSON(r.Context(), w, r, resolvedPath, begin, "application/vnd.ipld.dag-json", uint64(mc.DagJson)) + i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.DagJson)) return case "application/vnd.ipld.dag-cbor": + case "application/cbor": logger.Debugw("serving dag-cbor", "path", contentPath) - i.serveJSON(r.Context(), w, r, resolvedPath, begin, "application/vnd.ipld.dag-cbor", uint64(mc.DagCbor)) + i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.DagCbor)) return default: // catch-all for unsuported application/vnd.* err := fmt.Errorf("unsupported format %q", responseFormat) @@ -870,8 +872,12 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string] return "application/vnd.ipld.car", nil, nil case "dag-json": return "application/vnd.ipld.dag-json", nil, nil + case "json": + return "application/json", nil, nil case "dag-cbor": return "application/vnd.ipld.dag-cbor", nil, nil + case "cbor": + return "application/cbor", nil, nil } } // Browsers and other user agents will send Accept header with generic types like: @@ -879,7 +885,9 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string] // We only care about explciit, vendor-specific content-types. for _, accept := range r.Header.Values("Accept") { // respond to the very first ipld content type - if strings.HasPrefix(accept, "application/vnd.ipld") { + if strings.HasPrefix(accept, "application/vnd.ipld") || + strings.HasPrefix(accept, "application/json") || + strings.HasPrefix(accept, "application/cbor") { mediatype, params, err := mime.ParseMediaType(accept) if err != nil { return "", nil, err diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go new file mode 100644 index 00000000000..c21ccedab11 --- /dev/null +++ b/core/corehttp/gateway_handler_codec.go @@ -0,0 +1,58 @@ +package corehttp + +import ( + "context" + "fmt" + "html" + "net/http" + "time" + + ipldlegacy "github.com/ipfs/go-ipld-legacy" + ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipfs/kubo/tracing" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +var unixEpochTime = time.Unix(0, 0) + +func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, ctype string, codec uint64) { + ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("ctype", ctype))) + defer span.End() + + // Set Cache-Control and read optional Last-Modified time + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + + // Sets correct Last-Modified header. This code is borrowed from the standard + // library (net/http/server.go) as we cannot use serveFile. + if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } + + addContentDispositionHeader(w, r, contentPath) + w.Header().Set("Content-Type", ctype) + w.Header().Set("X-Content-Type-Options", "nosniff") + + obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid()) + if err != nil { + webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) + return + } + + universal, ok := obj.(ipldlegacy.UniversalNode) + if !ok { + webError(w, "todo", fmt.Errorf("%T is not a valid IPLD node", obj), http.StatusInternalServerError) + return + } + finalNode := universal.(ipld.Node) + + encoder, err := multicodec.LookupEncoder(codec) + if err != nil { + webError(w, "todo", err, http.StatusInternalServerError) + return + } + + _ = encoder(finalNode, w) +} diff --git a/core/corehttp/gateway_handler_json.go b/core/corehttp/gateway_handler_json.go deleted file mode 100644 index dc0959393c8..00000000000 --- a/core/corehttp/gateway_handler_json.go +++ /dev/null @@ -1,44 +0,0 @@ -package corehttp - -import ( - "context" - "fmt" - "html" - "net/http" - "time" - - ipldlegacy "github.com/ipfs/go-ipld-legacy" - ipath "github.com/ipfs/interface-go-ipfs-core/path" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/multicodec" -) - -// serveCAR returns a CAR stream for specific DAG+selector -func (i *gatewayHandler) serveJSON(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, begin time.Time, ctype string, codec uint64) { - // ctx, span := tracing.Span(ctx, "Gateway", "ServeCAR", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) - // defer span.End() - // ctx, cancel := context.WithCancel(ctx) - // defer cancel() - - obj, err := i.api.Dag().Get(r.Context(), resolvedPath.Cid()) - if err != nil { - webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) - return - } - - universal, ok := obj.(ipldlegacy.UniversalNode) - if !ok { - webError(w, "todo", fmt.Errorf("%T is not a valid IPLD node", obj), http.StatusInternalServerError) - return - } - finalNode := universal.(ipld.Node) - - encoder, err := multicodec.LookupEncoder(codec) - if err != nil { - webError(w, "todo", err, http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", ctype) - _ = encoder(finalNode, w) -} From b557181e5ee5ae09923d12a7e19ebc1e7ffcabd7 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 10 Oct 2022 12:22:55 +0200 Subject: [PATCH 03/32] fix: go cases don't flow automatically :) --- core/corehttp/gateway_handler.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index a918129693c..445238694a6 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -431,13 +431,11 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request carVersion := formatParams["version"] i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin) return - case "application/vnd.ipld.dag-json": - case "application/json": + case "application/vnd.ipld.dag-json", "application/json": logger.Debugw("serving dag-json", "path", contentPath) i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.DagJson)) return - case "application/vnd.ipld.dag-cbor": - case "application/cbor": + case "application/vnd.ipld.dag-cbor", "application/cbor": logger.Debugw("serving dag-cbor", "path", contentPath) i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.DagCbor)) return From 4104bb7ece71263b645df5737a1a4e48b3d4b64c Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 10 Oct 2022 12:23:06 +0200 Subject: [PATCH 04/32] test: add some dag-json and dag-cbor tests --- test/sharness/t0123-gateway-json-cbor.sh | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 test/sharness/t0123-gateway-json-cbor.sh diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh new file mode 100755 index 00000000000..a0fa4201fa9 --- /dev/null +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +test_description="Test HTTP Gateway DAG-JSON (application/vnd.ipld.dag-json) and DAG-CBOR (application/vnd.ipld.dag-cbor) Support" + +. lib/test-lib.sh + +test_init_ipfs +test_launch_ipfs_daemon_without_network + +test_expect_success "Add the test directory" ' + mkdir -p rootDir/ipfs && + mkdir -p rootDir/ipns && + mkdir -p rootDir/api && + mkdir -p rootDir/ą/ę && + echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt && + echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt && + echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt && + echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt && + DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) && + FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) && + FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size) + echo "$FILE_CID / $FILE_SIZE" +' + +test_codec () { + format=$1 + + test_expect_success "GET $format with format=dag-$format has expected Content-Type" ' + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_not_contain "Content-Type: application/$format" curl_output + ' + + test_expect_success "GET $format with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' + curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_not_contain "Content-Type: application/$format" curl_output + ' + + test_expect_success "GET $format with format=$format has expected Content-Type" ' + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 && + test_should_contain "Content-Type: application/$format" curl_output && + test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output + ' + + test_expect_success "GET $format with 'Accept: application/$format' has expected Content-Type" ' + curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && + test_should_contain "Content-Type: application/$format" curl_output && + test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output + ' + + test_expect_success "GET $format has expected output for file" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && + ipfs dag get --output-codec dag-$format $FILE_CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' + + test_expect_success "GET $format has expected output for directory" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output 2>&1 && + ipfs dag get --output-codec dag-$format $DIR_CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' + + test_expect_success "GET $format with format=dag-$format and format=$format produce same output" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output_1 2>&1 && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=$format" > curl_output_2 2>&1 && + test_cmp curl_output_1 curl_output_2 + ' +} + +test_codec "json" +test_codec "cbor" + +test_kill_ipfs_daemon + +test_done \ No newline at end of file From 25893c5f27e6ed374003d172c360c9dbaa42edf0 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 10 Oct 2022 12:35:20 +0200 Subject: [PATCH 05/32] test: improve names --- test/sharness/t0123-gateway-json-cbor.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index a0fa4201fa9..f94d977f3c4 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -23,53 +23,54 @@ test_expect_success "Add the test directory" ' ' test_codec () { - format=$1 + name=$1 + format=$2 - test_expect_success "GET $format with format=dag-$format has expected Content-Type" ' + test_expect_success "GET $name with format=dag-$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' - test_expect_success "GET $format with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' + test_expect_success "GET $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' - test_expect_success "GET $format with format=$format has expected Content-Type" ' + test_expect_success "GET $name with format=$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' - test_expect_success "GET $format with 'Accept: application/$format' has expected Content-Type" ' + test_expect_success "GET $name with 'Accept: application/$format' has expected Content-Type" ' curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' - test_expect_success "GET $format has expected output for file" ' + test_expect_success "GET $name has expected output for file" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && ipfs dag get --output-codec dag-$format $FILE_CID > ipfs_dag_get_output 2>&1 && test_cmp ipfs_dag_get_output curl_output ' - test_expect_success "GET $format has expected output for directory" ' + test_expect_success "GET $name has expected output for directory" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output 2>&1 && ipfs dag get --output-codec dag-$format $DIR_CID > ipfs_dag_get_output 2>&1 && test_cmp ipfs_dag_get_output curl_output ' - test_expect_success "GET $format with format=dag-$format and format=$format produce same output" ' + test_expect_success "GET $name with format=dag-$format and format=$format produce same output" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output_1 2>&1 && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=$format" > curl_output_2 2>&1 && test_cmp curl_output_1 curl_output_2 ' } -test_codec "json" -test_codec "cbor" +test_codec "DAG-JSON" "json" +test_codec "DAG-CBOR" "cbor" test_kill_ipfs_daemon From e621e64ad915fc168908394b10109cf1d7187577 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 11 Oct 2022 11:46:57 +0200 Subject: [PATCH 06/32] feat: allow json and cbor data types too --- core/corehttp/gateway_handler.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 445238694a6..d993a5f662f 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -431,11 +431,25 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request carVersion := formatParams["version"] i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin) return - case "application/vnd.ipld.dag-json", "application/json": + case "application/json": + if resolvedPath.Cid().Prefix().Codec == uint64(mc.Json) { + logger.Debugw("serving json", "path", contentPath) + i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.Json)) + return + } + fallthrough + case "application/vnd.ipld.dag-json": logger.Debugw("serving dag-json", "path", contentPath) i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.DagJson)) return - case "application/vnd.ipld.dag-cbor", "application/cbor": + case "application/cbor": + if resolvedPath.Cid().Prefix().Codec == uint64(mc.Cbor) { + logger.Debugw("serving cbor", "path", contentPath) + i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.Cbor)) + return + } + fallthrough + case "application/vnd.ipld.dag-cbor": logger.Debugw("serving dag-cbor", "path", contentPath) i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.DagCbor)) return From 44946ed008bfda7169c62ed94540611515dcf638 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 13 Oct 2022 14:22:20 +0200 Subject: [PATCH 07/32] refactor: avoid encoding things that are already on their right encoding --- core/corehttp/gateway_handler.go | 28 ++++--------------- core/corehttp/gateway_handler_codec.go | 37 +++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index d993a5f662f..3b3afe9fbad 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -26,7 +26,6 @@ import ( coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" routing "github.com/libp2p/go-libp2p/core/routing" - mc "github.com/multiformats/go-multicodec" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -431,29 +430,12 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request carVersion := formatParams["version"] i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin) return - case "application/json": - if resolvedPath.Cid().Prefix().Codec == uint64(mc.Json) { - logger.Debugw("serving json", "path", contentPath) - i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.Json)) - return - } - fallthrough - case "application/vnd.ipld.dag-json": - logger.Debugw("serving dag-json", "path", contentPath) - i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.DagJson)) - return - case "application/cbor": - if resolvedPath.Cid().Prefix().Codec == uint64(mc.Cbor) { - logger.Debugw("serving cbor", "path", contentPath) - i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.Cbor)) - return - } - fallthrough - case "application/vnd.ipld.dag-cbor": - logger.Debugw("serving dag-cbor", "path", contentPath) - i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat, uint64(mc.DagCbor)) + case "application/json", "application/vnd.ipld.dag-json", + "application/cbor", "application/vnd.ipld.dag-cbor": + logger.Debugw("serving codec", "format", responseFormat, "path", contentPath) + i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) return - default: // catch-all for unsuported application/vnd.* + default: // catch-all for unsupported application/vnd.* err := fmt.Errorf("unsupported format %q", responseFormat) webError(w, "failed respond with requested content type", err, http.StatusBadRequest) return diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index c21ccedab11..6fa8a6b3afa 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -12,16 +12,36 @@ import ( "github.com/ipfs/kubo/tracing" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/multicodec" + mc "github.com/multiformats/go-multicodec" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) var unixEpochTime = time.Unix(0, 0) -func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, ctype string, codec uint64) { - ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("ctype", ctype))) +// contentTypeToCodecs maps the HTTP Content Type to the respective +// possible codecs. If the original data is in one of those codecs, +// we stream the raw bytes. Otherwise, we encode in the last codec +// of the list. +var contentTypeToCodecs = map[string][]uint64{ + "application/json": {uint64(mc.Json), uint64(mc.DagJson)}, + "application/vnd.ipld.dag-json": {uint64(mc.DagJson)}, + "application/cbor": {uint64(mc.Cbor), uint64(mc.DagCbor)}, + "application/vnd.ipld.dag-cbor": {uint64(mc.DagCbor)}, +} + +func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, contentType string) { + ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("contentType", contentType))) defer span.End() + codecs, ok := contentTypeToCodecs[contentType] + if !ok { + // This is never supposed to happen unless function is called with wrong parameters. + err := fmt.Errorf("unsupported content type: %s", contentType) + webError(w, err.Error(), err, http.StatusInternalServerError) + return + } + // Set Cache-Control and read optional Last-Modified time modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) @@ -32,7 +52,7 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, } addContentDispositionHeader(w, r, contentPath) - w.Header().Set("Content-Type", ctype) + w.Header().Set("Content-Type", contentType) w.Header().Set("X-Content-Type-Options", "nosniff") obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid()) @@ -41,6 +61,14 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } + // If the data is already encoded with the possible codecs, just stream it out. + for _, codec := range codecs { + if resolvedPath.Cid().Prefix().Codec == codec { + _, _ = w.Write(obj.RawData()) + return + } + } + universal, ok := obj.(ipldlegacy.UniversalNode) if !ok { webError(w, "todo", fmt.Errorf("%T is not a valid IPLD node", obj), http.StatusInternalServerError) @@ -48,7 +76,8 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, } finalNode := universal.(ipld.Node) - encoder, err := multicodec.LookupEncoder(codec) + // Otherwise convert it using the last codec of the list. + encoder, err := multicodec.LookupEncoder(codecs[len(codecs)-1]) if err != nil { webError(w, "todo", err, http.StatusInternalServerError) return From 199ab42168d5a3b46fa47f514c0e687ba1a6d1b9 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 13 Oct 2022 14:35:48 +0200 Subject: [PATCH 08/32] fix: remove responseFormat from logging --- core/corehttp/gateway_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 3b3afe9fbad..41a096c87a3 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -432,7 +432,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request return case "application/json", "application/vnd.ipld.dag-json", "application/cbor", "application/vnd.ipld.dag-cbor": - logger.Debugw("serving codec", "format", responseFormat, "path", contentPath) + logger.Debugw("serving codec", "path", contentPath) i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) return default: // catch-all for unsupported application/vnd.* From 89eb033098b28c86e526503449a2e630849ea1d4 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 13 Oct 2022 16:00:09 +0200 Subject: [PATCH 09/32] refactor: simplify serveCodec to use serveRawBlock iff data encoded in output encoding --- core/corehttp/gateway_handler.go | 2 +- core/corehttp/gateway_handler_block.go | 4 ++-- core/corehttp/gateway_handler_codec.go | 17 +++++++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 41a096c87a3..525ae2c1f98 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -423,7 +423,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request return case "application/vnd.ipld.raw": logger.Debugw("serving raw block", "path", contentPath) - i.serveRawBlock(r.Context(), w, r, resolvedPath, contentPath, begin) + i.serveRawBlock(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) return case "application/vnd.ipld.car": logger.Debugw("serving car stream", "path", contentPath) diff --git a/core/corehttp/gateway_handler_block.go b/core/corehttp/gateway_handler_block.go index 3bf7c76be2d..e3dff1c0ae2 100644 --- a/core/corehttp/gateway_handler_block.go +++ b/core/corehttp/gateway_handler_block.go @@ -14,7 +14,7 @@ import ( ) // serveRawBlock returns bytes behind a raw block -func (i *gatewayHandler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time) { +func (i *gatewayHandler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, contentType string) { ctx, span := tracing.Span(ctx, "Gateway", "ServeRawBlock", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() blockCid := resolvedPath.Cid() @@ -41,7 +41,7 @@ func (i *gatewayHandler) serveRawBlock(ctx context.Context, w http.ResponseWrite // Set remaining headers modtime := addCacheControlHeaders(w, r, contentPath, blockCid) - w.Header().Set("Content-Type", "application/vnd.ipld.raw") + w.Header().Set("Content-Type", contentType) w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) // ServeContent will take care of diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 6fa8a6b3afa..615c28e9279 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -42,6 +42,15 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } + // If the data is already encoded with the possible codecs, we can defer execution to + // serveRawBlock, which will simply stream the raw data of this block. + for _, codec := range codecs { + if resolvedPath.Cid().Prefix().Codec == codec { + i.serveRawBlock(ctx, w, r, resolvedPath, contentPath, begin, contentType) + return + } + } + // Set Cache-Control and read optional Last-Modified time modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) @@ -61,14 +70,6 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } - // If the data is already encoded with the possible codecs, just stream it out. - for _, codec := range codecs { - if resolvedPath.Cid().Prefix().Codec == codec { - _, _ = w.Write(obj.RawData()) - return - } - } - universal, ok := obj.(ipldlegacy.UniversalNode) if !ok { webError(w, "todo", fmt.Errorf("%T is not a valid IPLD node", obj), http.StatusInternalServerError) From fb508690df5521ecb9c6f97c81dfba9dbe544249 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 13 Oct 2022 16:00:27 +0200 Subject: [PATCH 10/32] tests: rename current tests to indicate they're unixfs only --- test/sharness/t0123-gateway-json-cbor.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index f94d977f3c4..ec4b28f6460 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -22,55 +22,55 @@ test_expect_success "Add the test directory" ' echo "$FILE_CID / $FILE_SIZE" ' -test_codec () { +test_codec_unixfs () { name=$1 format=$2 - test_expect_success "GET $name with format=dag-$format has expected Content-Type" ' + test_expect_success "GET UnixFS $name with format=dag-$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' - test_expect_success "GET $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' + test_expect_success "GET UnixFS $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' - test_expect_success "GET $name with format=$format has expected Content-Type" ' + test_expect_success "GET UnixFS $name with format=$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' - test_expect_success "GET $name with 'Accept: application/$format' has expected Content-Type" ' + test_expect_success "GET UnixFS $name with 'Accept: application/$format' has expected Content-Type" ' curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' - test_expect_success "GET $name has expected output for file" ' + test_expect_success "GET UnixFS $name has expected output for file" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && ipfs dag get --output-codec dag-$format $FILE_CID > ipfs_dag_get_output 2>&1 && test_cmp ipfs_dag_get_output curl_output ' - test_expect_success "GET $name has expected output for directory" ' + test_expect_success "GET UnixFS $name has expected output for directory" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output 2>&1 && ipfs dag get --output-codec dag-$format $DIR_CID > ipfs_dag_get_output 2>&1 && test_cmp ipfs_dag_get_output curl_output ' - test_expect_success "GET $name with format=dag-$format and format=$format produce same output" ' + test_expect_success "GET UnixFS $name with format=dag-$format and format=$format produce same output" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output_1 2>&1 && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=$format" > curl_output_2 2>&1 && test_cmp curl_output_1 curl_output_2 ' } -test_codec "DAG-JSON" "json" -test_codec "DAG-CBOR" "cbor" +test_codec_unixfs "DAG-JSON" "json" +test_codec_unixfs "DAG-CBOR" "cbor" test_kill_ipfs_daemon From fc312413da55a1f3a1fc8cd723bbbc7caea85620 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 13 Oct 2022 16:52:14 +0200 Subject: [PATCH 11/32] refactor: do not use serveRawBlock inside serveCodec bc headers and other options --- core/corehttp/gateway_handler.go | 2 +- core/corehttp/gateway_handler_block.go | 4 +-- core/corehttp/gateway_handler_codec.go | 37 +++++++++++++++++++------- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 525ae2c1f98..41a096c87a3 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -423,7 +423,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request return case "application/vnd.ipld.raw": logger.Debugw("serving raw block", "path", contentPath) - i.serveRawBlock(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) + i.serveRawBlock(r.Context(), w, r, resolvedPath, contentPath, begin) return case "application/vnd.ipld.car": logger.Debugw("serving car stream", "path", contentPath) diff --git a/core/corehttp/gateway_handler_block.go b/core/corehttp/gateway_handler_block.go index e3dff1c0ae2..3bf7c76be2d 100644 --- a/core/corehttp/gateway_handler_block.go +++ b/core/corehttp/gateway_handler_block.go @@ -14,7 +14,7 @@ import ( ) // serveRawBlock returns bytes behind a raw block -func (i *gatewayHandler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, contentType string) { +func (i *gatewayHandler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time) { ctx, span := tracing.Span(ctx, "Gateway", "ServeRawBlock", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() blockCid := resolvedPath.Cid() @@ -41,7 +41,7 @@ func (i *gatewayHandler) serveRawBlock(ctx context.Context, w http.ResponseWrite // Set remaining headers modtime := addCacheControlHeaders(w, r, contentPath, blockCid) - w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Type", "application/vnd.ipld.raw") w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) // ServeContent will take care of diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 615c28e9279..7e01ab7923e 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -1,9 +1,11 @@ package corehttp import ( + "bytes" "context" "fmt" "html" + "io" "net/http" "time" @@ -42,28 +44,43 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } - // If the data is already encoded with the possible codecs, we can defer execution to - // serveRawBlock, which will simply stream the raw data of this block. + // Set Cache-Control and read optional Last-Modified time + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + name := addContentDispositionHeader(w, r, contentPath) + w.Header().Set("Content-Type", contentType) + w.Header().Set("X-Content-Type-Options", "nosniff") + + // If the data is already encoded with the possible codecs, we can just stream the raw + // data. serveRawBlock cannot be directly used here as it sets different headers. for _, codec := range codecs { if resolvedPath.Cid().Prefix().Codec == codec { - i.serveRawBlock(ctx, w, r, resolvedPath, contentPath, begin, contentType) + + blockCid := resolvedPath.Cid() + blockReader, err := i.api.Block().Get(ctx, resolvedPath) + if err != nil { + webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) + return + } + block, err := io.ReadAll(blockReader) + if err != nil { + webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) + return + } + content := bytes.NewReader(block) + + // ServeContent will take care of + // If-None-Match+Etag, Content-Length and range requests + _, _, _ = ServeContent(w, r, name, modtime, content) return } } - // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) - // Sets correct Last-Modified header. This code is borrowed from the standard // library (net/http/server.go) as we cannot use serveFile. if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) } - addContentDispositionHeader(w, r, contentPath) - w.Header().Set("Content-Type", contentType) - w.Header().Set("X-Content-Type-Options", "nosniff") - obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid()) if err != nil { webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) From 55383cdd673fb3d21553f62e7a45ccc067a4425a Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 17 Oct 2022 13:02:24 +0200 Subject: [PATCH 12/32] test: add test with pure json and cbor --- test/sharness/t0123-gateway-json-cbor.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index ec4b28f6460..7bf312b1744 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -72,6 +72,28 @@ test_codec_unixfs () { test_codec_unixfs "DAG-JSON" "json" test_codec_unixfs "DAG-CBOR" "cbor" +test_codec () { + name=$1 + format=$2 + + test_expect_success "GET $name with format=$format produces correct output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 && + ipfs dag get --output-codec $format $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' + + test_expect_success "GET $name with format=dag-$format produces correct output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" > curl_output 2>&1 && + ipfs dag get --output-codec dag-$format $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' +} + +test_codec "JSON" "json" +test_codec "CBOR" "cbor" + test_kill_ipfs_daemon test_done \ No newline at end of file From a6d45c775226d45cb41ad8b34d9fc7bffa559b63 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 17 Oct 2022 13:08:09 +0200 Subject: [PATCH 13/32] test: convert cbor <-> json --- test/sharness/t0123-gateway-json-cbor.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 7bf312b1744..ac446c3df7f 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -94,6 +94,20 @@ test_codec () { test_codec "JSON" "json" test_codec "CBOR" "cbor" +test_expect_success "GET JSON as CBOR produces DAG-CBOR output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec json) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=cbor" > curl_output 2>&1 && + ipfs dag get --output-codec dag-cbor $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output +' + +test_expect_success "GET CBOR as JSON produces DAG-JSON output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec cbor) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=json" > curl_output 2>&1 && + ipfs dag get --output-codec dag-json $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output +' + test_kill_ipfs_daemon test_done \ No newline at end of file From 1986be1cacf0945b0276e6b63251b3e0dd325de2 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 19 Oct 2022 15:49:18 +0200 Subject: [PATCH 14/32] test: path traversal and dag-pb output --- test/sharness/t0123-gateway-json-cbor.sh | 67 ++++++++++++++---- .../dag-cbor-traversal.car | Bin 0 -> 288 bytes .../dag-json-traversal.car | Bin 0 -> 358 bytes .../t0123-gateway-json-cbor/dag-pb.car | Bin 0 -> 392 bytes .../t0123-gateway-json-cbor/dag-pb.json | 23 ++++++ 5 files changed, 77 insertions(+), 13 deletions(-) create mode 100644 test/sharness/t0123-gateway-json-cbor/dag-cbor-traversal.car create mode 100644 test/sharness/t0123-gateway-json-cbor/dag-json-traversal.car create mode 100644 test/sharness/t0123-gateway-json-cbor/dag-pb.car create mode 100644 test/sharness/t0123-gateway-json-cbor/dag-pb.json diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index ac446c3df7f..831b5f82ead 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -22,57 +22,65 @@ test_expect_success "Add the test directory" ' echo "$FILE_CID / $FILE_SIZE" ' -test_codec_unixfs () { +test_headers () { name=$1 format=$2 - test_expect_success "GET UnixFS $name with format=dag-$format has expected Content-Type" ' + test_expect_success "GET $name with format=dag-$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' - test_expect_success "GET UnixFS $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' + test_expect_success "GET $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' - test_expect_success "GET UnixFS $name with format=$format has expected Content-Type" ' + test_expect_success "GET $name with format=$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' - test_expect_success "GET UnixFS $name with 'Accept: application/$format' has expected Content-Type" ' + test_expect_success "GET $name with 'Accept: application/$format' has expected Content-Type" ' curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' +} + +test_headers "DAG-JSON" "json" +test_headers "DAG-CBOR" "cbor" - test_expect_success "GET UnixFS $name has expected output for file" ' +test_dag_pb () { + name=$1 + format=$2 + + test_expect_success "GET DAG-PB $name has expected output for file" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && ipfs dag get --output-codec dag-$format $FILE_CID > ipfs_dag_get_output 2>&1 && test_cmp ipfs_dag_get_output curl_output ' - test_expect_success "GET UnixFS $name has expected output for directory" ' + test_expect_success "GET DAG-PB $name has expected output for directory" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output 2>&1 && ipfs dag get --output-codec dag-$format $DIR_CID > ipfs_dag_get_output 2>&1 && test_cmp ipfs_dag_get_output curl_output ' - test_expect_success "GET UnixFS $name with format=dag-$format and format=$format produce same output" ' + test_expect_success "GET DAG-PB $name with format=dag-$format and format=$format produce same output" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output_1 2>&1 && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=$format" > curl_output_2 2>&1 && test_cmp curl_output_1 curl_output_2 ' } -test_codec_unixfs "DAG-JSON" "json" -test_codec_unixfs "DAG-CBOR" "cbor" +test_dag_pb "DAG-JSON" "json" +test_dag_pb "DAG-CBOR" "cbor" -test_codec () { +test_cmp_dag_get () { name=$1 format=$2 @@ -91,8 +99,8 @@ test_codec () { ' } -test_codec "JSON" "json" -test_codec "CBOR" "cbor" +test_cmp_dag_get "JSON" "json" +test_cmp_dag_get "CBOR" "cbor" test_expect_success "GET JSON as CBOR produces DAG-CBOR output" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec json) && @@ -108,6 +116,39 @@ test_expect_success "GET CBOR as JSON produces DAG-JSON output" ' test_cmp ipfs_dag_get_output curl_output ' +DAG_CBOR_TRAVERSAL_CID="bafyreiehxu373cu3v5gyxyxfsfjryscs7sq6fh3unqcqgqhdfn3n43vrgu" +DAG_JSON_TRAVERSAL_CID="baguqeeraoaeabj5hdfcmpkzfeiwtfwb3qbvfwzbiknqn7itcwsb2fdtu7eta" +DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke" + +test_expect_success "Add CARs for path traversal and DAG-PB representation tests" ' + ipfs dag import ../t0123-gateway-json-cbor/dag-cbor-traversal.car > import_output && + test_should_contain $DAG_CBOR_TRAVERSAL_CID import_output && + ipfs dag import ../t0123-gateway-json-cbor/dag-json-traversal.car > import_output && + test_should_contain $DAG_JSON_TRAVERSAL_CID import_output && + ipfs dag import ../t0123-gateway-json-cbor/dag-pb.car > import_output && + test_should_contain $DAG_PB_CID import_output +' + +test_expect_success "GET DAG-JSON traverses multiple links" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=dag-json" > curl_output 2>&1 && + jq --sort-keys . curl_output > actual && + echo "{ \"hello\": \"this is not a link\" }" | jq --sort-keys . > expected && + test_cmp expected actual +' + +test_expect_success "GET DAG-CBOR traverses multiple links" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=dag-json" > curl_output 2>&1 && + jq --sort-keys . curl_output > actual && + echo "{ \"hello\": \"this is not a link\" }" | jq --sort-keys . > expected && + test_cmp expected actual +' + +test_expect_success "GET DAG-PB has expected output" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_PB_CID?format=dag-json" > curl_output 2>&1 && + jq --sort-keys . curl_output > actual && + test_cmp ../t0123-gateway-json-cbor/dag-pb.json actual +' + test_kill_ipfs_daemon test_done \ No newline at end of file diff --git a/test/sharness/t0123-gateway-json-cbor/dag-cbor-traversal.car b/test/sharness/t0123-gateway-json-cbor/dag-cbor-traversal.car new file mode 100644 index 0000000000000000000000000000000000000000..40a0cd4013aea796f43ebb6296467ddc6870bb6b GIT binary patch literal 288 zcmcColv%u~eY+n$ofs_R5%g!_qxmH{tjrFNwaf11 zZ8S|UOD!tS%+F&CB1ZMX7^PH1FSG4-(A7cOxGHCCzW&B;+b`afe|8{O f5>m7p9BePfmi1PAF6cQD3GV`(l3b}in literal 0 HcmV?d00001 diff --git a/test/sharness/t0123-gateway-json-cbor/dag-json-traversal.car b/test/sharness/t0123-gateway-json-cbor/dag-json-traversal.car new file mode 100644 index 0000000000000000000000000000000000000000..fa56006d5f4ec6a6bb3fd4e4b46afb7df2eccde7 GIT binary patch literal 358 zcmcCslvCux`kTPSFTXxW6cAOY@?> zlAmhnWvNBQnfZB)MMNpDR!YmySF);B(pR!lN=i&GElf=pc0V zwZ?Jz%WkJ>wzbd7+;qtLdw$5V(g!?$tAX|uVOW_|Wt5*$T9H$fmY$erTwGCi+sBr4=&=4C6@0suiPjIsa# literal 0 HcmV?d00001 diff --git a/test/sharness/t0123-gateway-json-cbor/dag-pb.car b/test/sharness/t0123-gateway-json-cbor/dag-pb.car new file mode 100644 index 0000000000000000000000000000000000000000..a6bb076c7e323e449dfe3f6290adbe7e3826cf21 GIT binary patch literal 392 zcmcColv6N9G>c!tg+FvpA8Lzlk~eSLZrIRuTwwE~hlf;U z76qo4r4|)u=I1eXF%qL(NS8|mq^|IQa`O8b2I)OECVzyMewgCAPNpIter34l*}|*5 zT4#fWnA7s}C0vCJfyzUL6rzjWn-?3Nly$m#PLi+f!g;?F8kf!m7^ME`R`a{7@j*j~ z9jIKdq@qNEmy3ymF`6hVpl07>^l?*Z@mjj;ut>RWcE+qL>A(DBr2-c2SUERp>UaIG uLhMP2MPRe}KxP|(+(Sr@M`}(^zK%k9eo;<}B9|dCUVv)x3~&op Date: Thu, 20 Oct 2022 12:50:44 +0200 Subject: [PATCH 15/32] fix: add more info about errors --- core/corehttp/gateway_handler_codec.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 7e01ab7923e..95defa956dd 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -89,7 +89,8 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, universal, ok := obj.(ipldlegacy.UniversalNode) if !ok { - webError(w, "todo", fmt.Errorf("%T is not a valid IPLD node", obj), http.StatusInternalServerError) + err = fmt.Errorf("%T is not a valid IPLD node", obj) + webError(w, err.Error(), err, http.StatusInternalServerError) return } finalNode := universal.(ipld.Node) @@ -97,7 +98,7 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, // Otherwise convert it using the last codec of the list. encoder, err := multicodec.LookupEncoder(codecs[len(codecs)-1]) if err != nil { - webError(w, "todo", err, http.StatusInternalServerError) + webError(w, err.Error(), err, http.StatusInternalServerError) return } From cadc6812e8172a2a41a6a4dc043d2122c39434a9 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 20 Oct 2022 13:47:46 +0200 Subject: [PATCH 16/32] fix: add missing traversal --- core/corehttp/gateway_handler_codec.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 95defa956dd..42d5b56db14 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -14,6 +14,7 @@ import ( "github.com/ipfs/kubo/tracing" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/multicodec" + "github.com/ipld/go-ipld-prime/traversal" mc "github.com/multiformats/go-multicodec" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -95,6 +96,16 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, } finalNode := universal.(ipld.Node) + if len(resolvedPath.Remainder()) > 0 { + remainderPath := ipld.ParsePath(resolvedPath.Remainder()) + + finalNode, err = traversal.Get(finalNode, remainderPath) + if err != nil { + webError(w, err.Error(), err, http.StatusInternalServerError) + return + } + } + // Otherwise convert it using the last codec of the list. encoder, err := multicodec.LookupEncoder(codecs[len(codecs)-1]) if err != nil { From 2c93672515a982fc034291ad2497272f466e161b Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 10 Nov 2022 11:54:53 +0100 Subject: [PATCH 17/32] fix: remove duplicate variable --- core/corehttp/gateway_handler_codec.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 42d5b56db14..b9e9878acdd 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -20,8 +20,6 @@ import ( "go.opentelemetry.io/otel/trace" ) -var unixEpochTime = time.Unix(0, 0) - // contentTypeToCodecs maps the HTTP Content Type to the respective // possible codecs. If the original data is in one of those codecs, // we stream the raw bytes. Otherwise, we encode in the last codec From 1e844c56fd345e71421fcf857fdc88f3e92e3fd9 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 11 Nov 2022 13:15:11 +0100 Subject: [PATCH 18/32] refactor: do not support traversal --- core/corehttp/gateway_handler_codec.go | 21 ++++++++----------- test/sharness/t0123-gateway-json-cbor.sh | 26 +++++++++++++++--------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index b9e9878acdd..105510583c2 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -7,6 +7,7 @@ import ( "html" "io" "net/http" + "strings" "time" ipldlegacy "github.com/ipfs/go-ipld-legacy" @@ -14,7 +15,6 @@ import ( "github.com/ipfs/kubo/tracing" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/multicodec" - "github.com/ipld/go-ipld-prime/traversal" mc "github.com/multiformats/go-multicodec" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -43,6 +43,14 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } + // We do not support paths for non-UnixFS codecs. If we have a path, return 404. + path := strings.TrimSuffix(resolvedPath.String(), "/") + if strings.Count(path, "/") > 2 { + err := fmt.Errorf("%s not found", resolvedPath.String()) + webError(w, err.Error(), err, http.StatusNotFound) + return + } + // Set Cache-Control and read optional Last-Modified time modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) name := addContentDispositionHeader(w, r, contentPath) @@ -53,7 +61,6 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, // data. serveRawBlock cannot be directly used here as it sets different headers. for _, codec := range codecs { if resolvedPath.Cid().Prefix().Codec == codec { - blockCid := resolvedPath.Cid() blockReader, err := i.api.Block().Get(ctx, resolvedPath) if err != nil { @@ -94,16 +101,6 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, } finalNode := universal.(ipld.Node) - if len(resolvedPath.Remainder()) > 0 { - remainderPath := ipld.ParsePath(resolvedPath.Remainder()) - - finalNode, err = traversal.Get(finalNode, remainderPath) - if err != nil { - webError(w, err.Error(), err, http.StatusInternalServerError) - return - } - } - // Otherwise convert it using the last codec of the list. encoder, err := multicodec.LookupEncoder(codecs[len(codecs)-1]) if err != nil { diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 831b5f82ead..ce2dc6cda15 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -129,18 +129,24 @@ test_expect_success "Add CARs for path traversal and DAG-PB representation tests test_should_contain $DAG_PB_CID import_output ' -test_expect_success "GET DAG-JSON traverses multiple links" ' - curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=dag-json" > curl_output 2>&1 && - jq --sort-keys . curl_output > actual && - echo "{ \"hello\": \"this is not a link\" }" | jq --sort-keys . > expected && - test_cmp expected actual +test_expect_success "GET JSON traversal returns 404" ' + curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=json" > curl_output 2>&1 && + test_should_contain "404 Not Found" curl_output ' -test_expect_success "GET DAG-CBOR traverses multiple links" ' - curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=dag-json" > curl_output 2>&1 && - jq --sort-keys . curl_output > actual && - echo "{ \"hello\": \"this is not a link\" }" | jq --sort-keys . > expected && - test_cmp expected actual +test_expect_success "GET CBOR traversal returns 404" ' + curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=cbor" > curl_output 2>&1 && + test_should_contain "404 Not Found" curl_output +' + +test_expect_success "GET DAG-JSON traversal returns 404" ' + curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=dag-json" > curl_output 2>&1 && + test_should_contain "404 Not Found" curl_output +' + +test_expect_success "GET DAG-CBOR traversal returns 404" ' + curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=dag-json" > curl_output 2>&1 && + test_should_contain "404 Not Found" curl_output ' test_expect_success "GET DAG-PB has expected output" ' From bb98041d1ef0d9bb07718a20d33bd84b50c84196 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 15 Nov 2022 15:21:51 +0100 Subject: [PATCH 19/32] Update core/corehttp/gateway_handler_codec.go Co-authored-by: Marcin Rataj --- core/corehttp/gateway_handler_codec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 105510583c2..0edbd753c8e 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -57,7 +57,7 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, w.Header().Set("Content-Type", contentType) w.Header().Set("X-Content-Type-Options", "nosniff") - // If the data is already encoded with the possible codecs, we can just stream the raw + // If the data is already encoded with supported and compatible codec, we can just stream the raw // data. serveRawBlock cannot be directly used here as it sets different headers. for _, codec := range codecs { if resolvedPath.Cid().Prefix().Codec == codec { From 53d5878360d0aeb315fba06140a7c59cb1ddeea2 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 16 Nov 2022 12:05:17 +0100 Subject: [PATCH 20/32] improve PR to match spec --- ...ammeoi5llohicdvj5rvfwibwlinciwdbnixi.block | Bin 0 -> 46 bytes core/corehttp/gateway_handler.go | 13 +- core/corehttp/gateway_handler_codec.go | 148 +++++++++++++----- test/sharness/t0123-gateway-json-cbor.sh | 72 +++++---- .../dag-cbor-traversal.car | Bin 288 -> 318 bytes .../dag-json-traversal.car | Bin 358 -> 406 bytes 6 files changed, 163 insertions(+), 70 deletions(-) create mode 100644 bafyreigcxa2x5mifam2zlpammeoi5llohicdvj5rvfwibwlinciwdbnixi.block diff --git a/bafyreigcxa2x5mifam2zlpammeoi5llohicdvj5rvfwibwlinciwdbnixi.block b/bafyreigcxa2x5mifam2zlpammeoi5llohicdvj5rvfwibwlinciwdbnixi.block new file mode 100644 index 0000000000000000000000000000000000000000..bc15f35a2bd5d2d57b32710d1eb5b5d40bb36967 GIT binary patch literal 46 zcmV+}0MY-UV`gt}*eX~h00C7JAY~j@qC<@3+j?UDKned8Y~_r;kC)7Y|Dxx_Ek++Y E=f}bood5s; literal 0 HcmV?d00001 diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index d763fbdedd8..1222b17bcd7 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -26,6 +26,7 @@ import ( coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" routing "github.com/libp2p/go-libp2p/core/routing" + mc "github.com/multiformats/go-multicodec" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -417,9 +418,15 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request // Support custom response formats passed via ?format or Accept HTTP header switch responseFormat { - case "": // The implicit response format is UnixFS - logger.Debugw("serving unixfs", "path", contentPath) - i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + case "": + switch resolvedPath.Cid().Prefix().Codec { + case uint64(mc.Json), uint64(mc.DagJson), uint64(mc.Cbor), uint64(mc.DagCbor): + logger.Debugw("serving codec", "path", contentPath) + i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) + default: + logger.Debugw("serving unixfs", "path", contentPath) + i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + } return case "application/vnd.ipld.raw": logger.Debugw("serving raw block", "path", contentPath) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 0edbd753c8e..ba1e030b2fd 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -20,6 +20,15 @@ import ( "go.opentelemetry.io/otel/trace" ) +// codecToContentType maps the supported IPLD codecs to the HTTP Content +// Type they should have. +var codecToContentType = map[uint64]string{ + uint64(mc.Json): "application/json", + uint64(mc.Cbor): "application/cbor", + uint64(mc.DagJson): "application/vnd.ipld.dag-json", + uint64(mc.DagCbor): "application/vnd.ipld.dag-cbor", +} + // contentTypeToCodecs maps the HTTP Content Type to the respective // possible codecs. If the original data is in one of those codecs, // we stream the raw bytes. Otherwise, we encode in the last codec @@ -31,62 +40,102 @@ var contentTypeToCodecs = map[string][]uint64{ "application/vnd.ipld.dag-cbor": {uint64(mc.DagCbor)}, } -func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, contentType string) { - ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("contentType", contentType))) +func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, requestedContentType string) { + ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) defer span.End() - codecs, ok := contentTypeToCodecs[contentType] + // If the resolved path still has some remainder, return bad request. + if resolvedPath.Remainder() != "" { + path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder()) + err := fmt.Errorf("%s could not be fully resolved, try %s instead", resolvedPath.String(), path) + webError(w, "path has remainder", err, http.StatusBadRequest) + return + } + + // No content type is specified by the user (via Accept, or format=). However, + // we support this format. Let's handle it. + if requestedContentType == "" { + cidCodec := resolvedPath.Cid().Prefix().Codec + isDAG := cidCodec == uint64(mc.DagJson) || cidCodec == uint64(mc.DagCbor) + acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html") + + if isDAG && acceptsHTML { + i.serverCodecHTML(ctx, w, r, resolvedPath, contentPath) + } else { + cidContentType, ok := codecToContentType[cidCodec] + if !ok { + // Should not happen unless function is called with wrong parameters. + err := fmt.Errorf("content type not found for codec: %v", cidCodec) + webError(w, "internal error", err, http.StatusInternalServerError) + return + } + + i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, cidContentType) + } + + return + } + + // Otherwise, the user has requested a specific content type. Let's first get + // the codecs that can be used with this content type. + codecs, ok := contentTypeToCodecs[requestedContentType] if !ok { // This is never supposed to happen unless function is called with wrong parameters. - err := fmt.Errorf("unsupported content type: %s", contentType) + err := fmt.Errorf("unsupported content type: %s", requestedContentType) webError(w, err.Error(), err, http.StatusInternalServerError) return } - // We do not support paths for non-UnixFS codecs. If we have a path, return 404. - path := strings.TrimSuffix(resolvedPath.String(), "/") - if strings.Count(path, "/") > 2 { - err := fmt.Errorf("%s not found", resolvedPath.String()) - webError(w, err.Error(), err, http.StatusNotFound) + // If the requested content type has "dag-", ALWAYS go through the encoding + // process in order to validate the content. + if strings.Contains(requestedContentType, "dag-") { + i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1]) return } - // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) - name := addContentDispositionHeader(w, r, contentPath) - w.Header().Set("Content-Type", contentType) - w.Header().Set("X-Content-Type-Options", "nosniff") - - // If the data is already encoded with supported and compatible codec, we can just stream the raw - // data. serveRawBlock cannot be directly used here as it sets different headers. + // Otherwise, check if the data is encoded with the requested content type. + // If so, we can directly stream the raw data. serveRawBlock cannot be directly + // used here as it sets different headers. for _, codec := range codecs { if resolvedPath.Cid().Prefix().Codec == codec { - blockCid := resolvedPath.Cid() - blockReader, err := i.api.Block().Get(ctx, resolvedPath) - if err != nil { - webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) - return - } - block, err := io.ReadAll(blockReader) - if err != nil { - webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) - return - } - content := bytes.NewReader(block) - - // ServeContent will take care of - // If-None-Match+Etag, Content-Length and range requests - _, _, _ = ServeContent(w, r, name, modtime, content) + i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, requestedContentType) return } } - // Sets correct Last-Modified header. This code is borrowed from the standard - // library (net/http/server.go) as we cannot use serveFile. - if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { - w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + // Finally, if nothing of the above is true, we have to actually convert the codec. + i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1]) +} + +func (i *gatewayHandler) serverCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { + w.Write([]byte("TODO")) +} + +func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string) { + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + name := addContentDispositionHeader(w, r, contentPath) + w.Header().Set("Content-Type", contentType) + w.Header().Set("X-Content-Type-Options", "nosniff") + + blockCid := resolvedPath.Cid() + blockReader, err := i.api.Block().Get(ctx, resolvedPath) + if err != nil { + webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) + return + } + block, err := io.ReadAll(blockReader) + if err != nil { + webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) + return } + content := bytes.NewReader(block) + // ServeContent will take care of + // If-None-Match+Etag, Content-Length and range requests + _, _, _ = ServeContent(w, r, name, modtime, content) +} + +func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string, codec uint64) { obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid()) if err != nil { webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) @@ -101,12 +150,31 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, } finalNode := universal.(ipld.Node) - // Otherwise convert it using the last codec of the list. - encoder, err := multicodec.LookupEncoder(codecs[len(codecs)-1]) + encoder, err := multicodec.LookupEncoder(codec) + if err != nil { + webError(w, err.Error(), err, http.StatusInternalServerError) + return + } + + // Keep it in memory so we can detect encoding errors in order to conform + // to the specification. + var buf bytes.Buffer + err = encoder(finalNode, &buf) if err != nil { webError(w, err.Error(), err, http.StatusInternalServerError) return } - _ = encoder(finalNode, w) + // Set Cache-Control and read optional Last-Modified time + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + w.Header().Set("Content-Type", contentType) + w.Header().Set("X-Content-Type-Options", "nosniff") + + // Sets correct Last-Modified header. This code is borrowed from the standard + // library (net/http/server.go) as we cannot use serveFile. + if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } + + w.Write(buf.Bytes()) } diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index ce2dc6cda15..20160ecbaeb 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -26,25 +26,25 @@ test_headers () { name=$1 format=$2 - test_expect_success "GET $name with format=dag-$format has expected Content-Type" ' + test_expect_success "GET UnixFS as $name with format=dag-$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' - test_expect_success "GET $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' + test_expect_success "GET UnixFS as $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' - test_expect_success "GET $name with format=$format has expected Content-Type" ' + test_expect_success "GET UnixFS as $name with format=$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' - test_expect_success "GET $name with 'Accept: application/$format' has expected Content-Type" ' + test_expect_success "GET UnixFS as $name with 'Accept: application/$format' has expected Content-Type" ' curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output @@ -58,19 +58,19 @@ test_dag_pb () { name=$1 format=$2 - test_expect_success "GET DAG-PB $name has expected output for file" ' + test_expect_success "GET UnixFS as $name has expected output for file" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && ipfs dag get --output-codec dag-$format $FILE_CID > ipfs_dag_get_output 2>&1 && test_cmp ipfs_dag_get_output curl_output ' - test_expect_success "GET DAG-PB $name has expected output for directory" ' + test_expect_success "GET UnixFS as $name has expected output for directory" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output 2>&1 && ipfs dag get --output-codec dag-$format $DIR_CID > ipfs_dag_get_output 2>&1 && test_cmp ipfs_dag_get_output curl_output ' - test_expect_success "GET DAG-PB $name with format=dag-$format and format=$format produce same output" ' + test_expect_success "GET UnixFS as $name with format=dag-$format and format=$format produce same output" ' curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output_1 2>&1 && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=$format" > curl_output_2 2>&1 && test_cmp curl_output_1 curl_output_2 @@ -84,6 +84,19 @@ test_cmp_dag_get () { name=$1 format=$2 + test_expect_success "GET $name without Accept or format= has expected Content-Type" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + test_should_contain "Content-Type: application/$format" curl_output + ' + + test_expect_success "GET $name without Accept or format= produces correct output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + ipfs dag get --output-codec $format $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' + test_expect_success "GET $name with format=$format produces correct output" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 && @@ -116,8 +129,8 @@ test_expect_success "GET CBOR as JSON produces DAG-JSON output" ' test_cmp ipfs_dag_get_output curl_output ' -DAG_CBOR_TRAVERSAL_CID="bafyreiehxu373cu3v5gyxyxfsfjryscs7sq6fh3unqcqgqhdfn3n43vrgu" -DAG_JSON_TRAVERSAL_CID="baguqeeraoaeabj5hdfcmpkzfeiwtfwb3qbvfwzbiknqn7itcwsb2fdtu7eta" +DAG_CBOR_TRAVERSAL_CID="bafyreibs4utpgbn7uqegmd2goqz4bkyflre2ek2iwv743fhvylwi4zeeim" +DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxkd47xq" DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke" test_expect_success "Add CARs for path traversal and DAG-PB representation tests" ' @@ -129,32 +142,37 @@ test_expect_success "Add CARs for path traversal and DAG-PB representation tests test_should_contain $DAG_PB_CID import_output ' -test_expect_success "GET JSON traversal returns 404" ' - curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=json" > curl_output 2>&1 && - test_should_contain "404 Not Found" curl_output -' - -test_expect_success "GET CBOR traversal returns 404" ' - curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=cbor" > curl_output 2>&1 && - test_should_contain "404 Not Found" curl_output +test_expect_success "GET DAG-JSON traversal returns 400 if there is path remainder" ' + curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo?format=dag-json" > curl_output 2>&1 && + test_should_contain "400 Bad Request" curl_output ' -test_expect_success "GET DAG-JSON traversal returns 404" ' - curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=dag-json" > curl_output 2>&1 && - test_should_contain "404 Not Found" curl_output +test_expect_success "GET DAG-JSON traverses multiple links" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo/link/bar?format=dag-json" > curl_output 2>&1 && + jq --sort-keys . curl_output > actual && + echo "{ \"hello\": \"this is not a link\" }" | jq --sort-keys . > expected && + test_cmp expected actual ' -test_expect_success "GET DAG-CBOR traversal returns 404" ' - curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/bar?format=dag-json" > curl_output 2>&1 && - test_should_contain "404 Not Found" curl_output +test_expect_success "GET DAG-CBOR traversal returns 400 if there is path remainder" ' + curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo?format=dag-cbor" > curl_output 2>&1 && + test_should_contain "400 Bad Request" curl_output ' -test_expect_success "GET DAG-PB has expected output" ' - curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_PB_CID?format=dag-json" > curl_output 2>&1 && +test_expect_success "GET DAG-CBOR traverses multiple links" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/link/bar?format=dag-json" > curl_output 2>&1 && jq --sort-keys . curl_output > actual && - test_cmp ../t0123-gateway-json-cbor/dag-pb.json actual + echo "{ \"hello\": \"this is not a link\" }" | jq --sort-keys . > expected && + test_cmp expected actual ' +# test_expect_success "GET DAG-PB has expected output" ' +# curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_PB_CID?format=dag-json" > curl_output 2>&1 && +# jq --sort-keys . curl_output > actual && +# test_cmp ../t0123-gateway-json-cbor/dag-pb.json actual +# ' + test_kill_ipfs_daemon -test_done \ No newline at end of file +test_done + diff --git a/test/sharness/t0123-gateway-json-cbor/dag-cbor-traversal.car b/test/sharness/t0123-gateway-json-cbor/dag-cbor-traversal.car index 40a0cd4013aea796f43ebb6296467ddc6870bb6b..92c3d4f3e21718627007431c6b6d0cc1693be206 100644 GIT binary patch delta 163 zcmZ3$w2!IYYEf!Yett=D;|;9{RR+dFAqAtSYM)v6FX2ezcPlYIu$nc-bCI^k*7~zk zz8-qhm(t>#UY1%^oSC1;SU`;Gg~@68`HNC=GV`+GHr!=8p^;g|)WdRLY~nmoiM0Hr btkmR^MQKTic|gdOmYbiKom%;2V&5bHTlhm< delta 117 zcmdnTw16qqYEf!Yett=D;|;9{RR+dFA%*t6=6}0pulMbK^mJmdj7QL)g^%W!l!7XZzkGyVVo diff --git a/test/sharness/t0123-gateway-json-cbor/dag-json-traversal.car b/test/sharness/t0123-gateway-json-cbor/dag-json-traversal.car index fa56006d5f4ec6a6bb3fd4e4b46afb7df2eccde7..0431ef24aea2872d2c7113a1048c79373e50ed2e 100644 GIT binary patch delta 177 zcmaFHG>y65dQoaoett=D;|;9{H3r6&OhO9j8B-3KSsHLYE_>OoW^K1h-B50M_}4AV z#KX_A#yUTLpI(+)RGgWg$GDJ@DD~A!Y5Dm|R@F*5nR(eDT3^XZDJe0%v@kVwVwr`S qj#7S7R%&tyNGd5Y4+xd43=MRYa`W@DQ!ACM(h_ruQ)?%#-3b7?o_9ZIN|=H kq%F;h`bvJPrI)1^6=&w>F%}V}yjm$OKYwC@`^0-Y0JQfltpET3 From 8c6a8daccd5f294e185d61648167f43e080680d0 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 16 Nov 2022 12:24:44 +0100 Subject: [PATCH 21/32] feat: little web page --- core/corehttp/gateway_handler_codec.go | 48 ++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index ba1e030b2fd..8f0a3c3fd95 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -40,6 +40,15 @@ var contentTypeToCodecs = map[string][]uint64{ "application/vnd.ipld.dag-cbor": {uint64(mc.DagCbor)}, } +// contentTypeToExtension maps the HTTP Content Type to the respective file +// extension, to apply them when downloading the file. +var contentTypeToExtension = map[string]string{ + "application/json": ".json", + "application/vnd.ipld.dag-json": ".dag-json", + "application/cbor": ".cbor", + "application/vnd.ipld.dag-cbor": ".dag-cbor", +} + func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, requestedContentType string) { ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) defer span.End() @@ -108,12 +117,30 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, } func (i *gatewayHandler) serverCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { - w.Write([]byte("TODO")) + body := fmt.Sprintf(` + + + + + +

The document you are trying to access cannot be previewed in the browser:

+ 
%s
+

Please follow the following links to download the document in other formats:

+ + + +`, contentPath.String()) + + w.Write([]byte(body)) } func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string) { modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) - name := addContentDispositionHeader(w, r, contentPath) + name := setCodecContentDisposition(w, r, resolvedPath, contentType) w.Header().Set("Content-Type", contentType) w.Header().Set("X-Content-Type-Options", "nosniff") @@ -167,6 +194,7 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons // Set Cache-Control and read optional Last-Modified time modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + setCodecContentDisposition(w, r, resolvedPath, contentType) w.Header().Set("Content-Type", contentType) w.Header().Set("X-Content-Type-Options", "nosniff") @@ -178,3 +206,19 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons w.Write(buf.Bytes()) } + +func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentType string) string { + var name string + if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" { + name = urlFilename + } else { + ext, ok := contentTypeToExtension[contentType] + if !ok { + // Should never happen. + ext = ".bin" + } + name = resolvedPath.Cid().String() + ext + } + setContentDispositionHeader(w, name, "attachment") + return name +} From b5e5ff20e562e42888a9e96c1257c149afe88bee Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 16 Nov 2022 12:26:07 +0100 Subject: [PATCH 22/32] feat: update doc --- core/corehttp/gateway_handler_codec.go | 10 +++++----- test/sharness/t0123-gateway-json-cbor.sh | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 8f0a3c3fd95..355c8047239 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -127,15 +127,15 @@ func (i *gatewayHandler) serverCodecHTML(ctx context.Context, w http.ResponseWri 
%s

Please follow the following links to download the document in other formats:

`, contentPath.String()) - w.Write([]byte(body)) + _, _ = w.Write([]byte(body)) } func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string) { @@ -204,7 +204,7 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) } - w.Write(buf.Bytes()) + _, _ = w.Write(buf.Bytes()) } func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentType string) string { diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 20160ecbaeb..3a64af64356 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -142,6 +142,11 @@ test_expect_success "Add CARs for path traversal and DAG-PB representation tests test_should_contain $DAG_PB_CID import_output ' +test_expect_success "GET DAG-JSON with Accept: text/html returns HTML" ' + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID" > curl_output 2>&1 && + test_should_contain "Content-Type: text/html" curl_output +' + test_expect_success "GET DAG-JSON traversal returns 400 if there is path remainder" ' curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo?format=dag-json" > curl_output 2>&1 && test_should_contain "400 Bad Request" curl_output @@ -154,6 +159,11 @@ test_expect_success "GET DAG-JSON traverses multiple links" ' test_cmp expected actual ' +test_expect_success "GET DAG-CBOR with Accept: text/html returns HTML" ' + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID" > curl_output 2>&1 && + test_should_contain "Content-Type: text/html" curl_output +' + test_expect_success "GET DAG-CBOR traversal returns 400 if there is path remainder" ' curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo?format=dag-cbor" > curl_output 2>&1 && test_should_contain "400 Bad Request" curl_output From 8ca2a52dc91f14bbbf979996dcd5c9aef4ecea11 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 23 Nov 2022 16:34:24 +0100 Subject: [PATCH 23/32] fix: Content-Disposition .json and .cbor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .dag- veriants are not necessary – DAG-JSON should have 'Save As' dialog default to file named .json see "File extension" section of specs below: https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-json https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-cbor --- core/corehttp/gateway_handler_codec.go | 6 +++--- test/sharness/t0123-gateway-json-cbor.sh | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 355c8047239..e9df9c9b5c4 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -41,12 +41,12 @@ var contentTypeToCodecs = map[string][]uint64{ } // contentTypeToExtension maps the HTTP Content Type to the respective file -// extension, to apply them when downloading the file. +// extension, used in Content-Disposition header when downloading the file. var contentTypeToExtension = map[string]string{ "application/json": ".json", - "application/vnd.ipld.dag-json": ".dag-json", + "application/vnd.ipld.dag-json": ".json", "application/cbor": ".cbor", - "application/vnd.ipld.dag-cbor": ".dag-cbor", + "application/vnd.ipld.dag-cbor": ".cbor", } func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, requestedContentType string) { diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 3a64af64356..9c45ce9a5d3 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -29,24 +29,28 @@ test_headers () { test_expect_success "GET UnixFS as $name with format=dag-$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_contain "Content-Disposition: attachment\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' test_expect_success "GET UnixFS as $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_contain "Content-Disposition: attachment\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' test_expect_success "GET UnixFS as $name with format=$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && + test_should_contain "Content-Disposition: attachment\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' test_expect_success "GET UnixFS as $name with 'Accept: application/$format' has expected Content-Type" ' curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && test_should_contain "Content-Type: application/$format" curl_output && + test_should_contain "Content-Disposition: attachment\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' } @@ -87,6 +91,7 @@ test_cmp_dag_get () { test_expect_success "GET $name without Accept or format= has expected Content-Type" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + test_should_contain "Content-Disposition: attachment\; filename=\"${CID}.${format}\"" curl_output && test_should_contain "Content-Type: application/$format" curl_output ' @@ -144,6 +149,7 @@ test_expect_success "Add CARs for path traversal and DAG-PB representation tests test_expect_success "GET DAG-JSON with Accept: text/html returns HTML" ' curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID" > curl_output 2>&1 && + test_should_not_contain "Content-Disposition: attachment" curl_output && test_should_contain "Content-Type: text/html" curl_output ' @@ -161,6 +167,7 @@ test_expect_success "GET DAG-JSON traverses multiple links" ' test_expect_success "GET DAG-CBOR with Accept: text/html returns HTML" ' curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID" > curl_output 2>&1 && + test_should_not_contain "Content-Disposition: attachment" curl_output && test_should_contain "Content-Type: text/html" curl_output ' From b4dfa662cc6b9315276221d6e795e4b851c0d908 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 23 Nov 2022 17:17:53 +0100 Subject: [PATCH 24/32] fix: inline disposition for JSON responses Quality of life improvement, allows user agents to render JSON. Passing ?format=json to unixfs directory will show JSON in browser. --- core/corehttp/gateway_handler_codec.go | 23 +++++++++----- test/sharness/t0123-gateway-json-cbor.sh | 38 +++++++++++++++++------- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index e9df9c9b5c4..391fb4dba40 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -208,17 +208,26 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons } func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentType string) string { - var name string + var dispType, name string + + ext, ok := contentTypeToExtension[contentType] + if !ok { + // Should never happen. + ext = ".bin" + } + if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" { name = urlFilename } else { - ext, ok := contentTypeToExtension[contentType] - if !ok { - // Should never happen. - ext = ".bin" - } name = resolvedPath.Cid().String() + ext } - setContentDispositionHeader(w, name, "attachment") + + switch ext { + case ".json": // codecs that serialize to JSON can be rendered by browsers + dispType = "inline" + default: // everything else is assumed binary / opaque bytes + dispType = "attachment" + } + setContentDispositionHeader(w, name, dispType) return name } diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 9c45ce9a5d3..7b49bb4c9e5 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -25,38 +25,39 @@ test_expect_success "Add the test directory" ' test_headers () { name=$1 format=$2 + disposition=$3 test_expect_success "GET UnixFS as $name with format=dag-$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && - test_should_contain "Content-Disposition: attachment\; filename=\"${FILE_CID}.${format}\"" curl_output && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' test_expect_success "GET UnixFS as $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && - test_should_contain "Content-Disposition: attachment\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_not_contain "Content-Type: application/$format" curl_output ' test_expect_success "GET UnixFS as $name with format=$format has expected Content-Type" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_contain "Content-Type: application/$format" curl_output && - test_should_contain "Content-Disposition: attachment\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' test_expect_success "GET UnixFS as $name with 'Accept: application/$format' has expected Content-Type" ' curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_contain "Content-Type: application/$format" curl_output && - test_should_contain "Content-Disposition: attachment\; filename=\"${FILE_CID}.${format}\"" curl_output && test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output ' } -test_headers "DAG-JSON" "json" -test_headers "DAG-CBOR" "cbor" +test_headers "DAG-JSON" "json" "inline" +test_headers "DAG-CBOR" "cbor" "attachment" test_dag_pb () { name=$1 @@ -87,11 +88,12 @@ test_dag_pb "DAG-CBOR" "cbor" test_cmp_dag_get () { name=$1 format=$2 + disposition=$3 test_expect_success "GET $name without Accept or format= has expected Content-Type" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && - test_should_contain "Content-Disposition: attachment\; filename=\"${CID}.${format}\"" curl_output && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output && test_should_contain "Content-Type: application/$format" curl_output ' @@ -102,6 +104,13 @@ test_cmp_dag_get () { test_cmp ipfs_dag_get_output curl_output ' + test_expect_success "GET $name with format=$format produces expected Content-Type" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -sD- "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output && + test_should_contain "Content-Type: application/$format" curl_output + ' + test_expect_success "GET $name with format=$format produces correct output" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 && @@ -109,6 +118,13 @@ test_cmp_dag_get () { test_cmp ipfs_dag_get_output curl_output ' + test_expect_success "GET $name with format=dag-$format produces expected Content-Type" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -sD- "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output + ' + test_expect_success "GET $name with format=dag-$format produces correct output" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" > curl_output 2>&1 && @@ -117,8 +133,8 @@ test_cmp_dag_get () { ' } -test_cmp_dag_get "JSON" "json" -test_cmp_dag_get "CBOR" "cbor" +test_cmp_dag_get "JSON" "json" "inline" +test_cmp_dag_get "CBOR" "cbor" "attachment" test_expect_success "GET JSON as CBOR produces DAG-CBOR output" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec json) && @@ -149,7 +165,7 @@ test_expect_success "Add CARs for path traversal and DAG-PB representation tests test_expect_success "GET DAG-JSON with Accept: text/html returns HTML" ' curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID" > curl_output 2>&1 && - test_should_not_contain "Content-Disposition: attachment" curl_output && + test_should_not_contain "Content-Disposition" curl_output && test_should_contain "Content-Type: text/html" curl_output ' @@ -167,7 +183,7 @@ test_expect_success "GET DAG-JSON traverses multiple links" ' test_expect_success "GET DAG-CBOR with Accept: text/html returns HTML" ' curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID" > curl_output 2>&1 && - test_should_not_contain "Content-Disposition: attachment" curl_output && + test_should_not_contain "Content-Disposition" curl_output && test_should_contain "Content-Type: text/html" curl_output ' From 83913c7647d7c50129bc1c66836653f4584dfe89 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 23 Nov 2022 17:37:07 +0100 Subject: [PATCH 25/32] refactor: return 501 for unsupported pathing Includes human-readable error which encourages people to use CBOR Tag 42 for building DAGs with Links. --- core/corehttp/gateway_handler_codec.go | 4 ++-- test/sharness/t0123-gateway-json-cbor.sh | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 391fb4dba40..8b6c1503562 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -56,8 +56,8 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, // If the resolved path still has some remainder, return bad request. if resolvedPath.Remainder() != "" { path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder()) - err := fmt.Errorf("%s could not be fully resolved, try %s instead", resolvedPath.String(), path) - webError(w, "path has remainder", err, http.StatusBadRequest) + err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", resolvedPath.Remainder(), resolvedPath.String(), path) + webError(w, "unsupported pathing", err, http.StatusNotImplemented) return } diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 7b49bb4c9e5..535249ab206 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -169,9 +169,10 @@ test_expect_success "GET DAG-JSON with Accept: text/html returns HTML" ' test_should_contain "Content-Type: text/html" curl_output ' -test_expect_success "GET DAG-JSON traversal returns 400 if there is path remainder" ' - curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo?format=dag-json" > curl_output 2>&1 && - test_should_contain "400 Bad Request" curl_output +test_expect_success "GET DAG-JSON traversal returns 501 if there is path remainder" ' + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo?format=dag-json" > curl_output 2>&1 && + test_should_contain "501 Not Implemented" curl_output && + test_should_contain "reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented" curl_output ' test_expect_success "GET DAG-JSON traverses multiple links" ' @@ -187,9 +188,10 @@ test_expect_success "GET DAG-CBOR with Accept: text/html returns HTML" ' test_should_contain "Content-Type: text/html" curl_output ' -test_expect_success "GET DAG-CBOR traversal returns 400 if there is path remainder" ' - curl --head "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo?format=dag-cbor" > curl_output 2>&1 && - test_should_contain "400 Bad Request" curl_output +test_expect_success "GET DAG-CBOR traversal returns 501 if there is path remainder" ' + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo?format=dag-cbor" > curl_output 2>&1 && + test_should_contain "501 Not Implemented" curl_output && + test_should_contain "reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented" curl_output ' test_expect_success "GET DAG-CBOR traverses multiple links" ' From ff5574527a10625d072c23cfbc6d1fa59bcb9f76 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 23 Nov 2022 18:53:35 +0100 Subject: [PATCH 26/32] docs(cbor): improved info about codec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This ensures user knows why they got this HTML response (due to specific codec in the specific CID they can inspect), and that they have clear understanding of options – either preview as JSON, or download as one of supported formats. --- core/corehttp/gateway_handler_codec.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 8b6c1503562..5c9ebe80858 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -69,7 +69,7 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html") if isDAG && acceptsHTML { - i.serverCodecHTML(ctx, w, r, resolvedPath, contentPath) + i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath) } else { cidContentType, ok := codecToContentType[cidCodec] if !ok { @@ -116,24 +116,28 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1]) } -func (i *gatewayHandler) serverCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { +func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { + codecName := mc.Code(resolvedPath.Cid().Prefix().Codec).String() body := fmt.Sprintf(` -

The document you are trying to access cannot be previewed in the browser:

- 
%s
-

Please follow the following links to download the document in other formats:

+

Requested CID %q uses %q codec.

-`, contentPath.String()) +`, resolvedPath.Cid(), codecName) _, _ = w.Write([]byte(body)) } From 52711d3fbc80e36e6adb9e5a11c4ce6e6cc13743 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 24 Nov 2022 17:39:52 +0100 Subject: [PATCH 27/32] refactor: create template at assets/dag-index-html This is a quick, static placeholder that follows the style of dir-index-html. Future work will refine this, perhaps by reusing parts of https://explore.ipld.io/ --- assets/dag-index-html/README.md | 3 + assets/dag-index-html/index.go | 82 ++++++++++++++++++++++ assets/dir-index-html/README.md | 2 +- assets/dir-index-html/dir-index.html | 12 ++-- assets/dir-index-html/src/dir-index.html | 12 ++-- core/corehttp/gateway_handler_codec.go | 34 +++------ core/corehttp/gateway_test.go | 8 +-- test/sharness/t0115-gateway-dir-listing.sh | 4 +- 8 files changed, 115 insertions(+), 42 deletions(-) create mode 100644 assets/dag-index-html/README.md create mode 100644 assets/dag-index-html/index.go diff --git a/assets/dag-index-html/README.md b/assets/dag-index-html/README.md new file mode 100644 index 00000000000..de38a9504a1 --- /dev/null +++ b/assets/dag-index-html/README.md @@ -0,0 +1,3 @@ +# dag-index-html + +> HTML representation for non-UnixFS DAGs such as DAG-CBOR. diff --git a/assets/dag-index-html/index.go b/assets/dag-index-html/index.go new file mode 100644 index 00000000000..dd1e93d799d --- /dev/null +++ b/assets/dag-index-html/index.go @@ -0,0 +1,82 @@ +package dagindexhtml + +import "html/template" + +// TODO: DagIndexTemplate - replace static CSS with shared one with ../dir-index-html + +// DagIndexTemplate is HTML-based template for non-UnixFS DAGs when request was +// made with Accept: text/html (web browsers). +var DagIndexTemplate = template.Must(template.New("redirect").Parse(` + + + + + + + + + + + + + + + + + + {{ .Path }} + + + +
+ +
+
About IPFS
+
Install IPFS
+
About
+
Install
+
+ + + +
+
+
+
+
+

CID: {{.CID}}
+ Codec: {{.CodecName}} ({{.CodecHex}})

+

This CID is not UnixFS (not a file, nor a directory)

+
+
+ + + + + + + +
+

Preview as JSON
(application/json)

+
+

Or download as: +

+

+
+
+
+ +`)) + +type DagIndexTemplateData struct { + Path string + CID string + CodecName string + CodecHex string +} diff --git a/assets/dir-index-html/README.md b/assets/dir-index-html/README.md index ad5fa68b2db..3dd45eb5905 100644 --- a/assets/dir-index-html/README.md +++ b/assets/dir-index-html/README.md @@ -1,6 +1,6 @@ # dir-index-html -> Directory listing HTML for `go-ipfs` gateways +> Directory listing HTML for HTTP gateway ![](https://user-images.githubusercontent.com/157609/88379209-ce6f0600-cda2-11ea-9620-20b9237bb441.png) diff --git a/assets/dir-index-html/dir-index.html b/assets/dir-index-html/dir-index.html index a2d662d19c7..1d00e5fe73f 100644 --- a/assets/dir-index-html/dir-index.html +++ b/assets/dir-index-html/dir-index.html @@ -26,12 +26,12 @@
-
About IPFS
-
Install IPFS
-
About
-
Install
+
About IPFS
+
Install IPFS
+
About
+
Install
@@ -84,7 +84,7 @@ {{ if .Hash }} - + {{ .ShortHash }} {{ end }} diff --git a/assets/dir-index-html/src/dir-index.html b/assets/dir-index-html/src/dir-index.html index 1bbbb0a9a8c..a763b3e76bb 100644 --- a/assets/dir-index-html/src/dir-index.html +++ b/assets/dir-index-html/src/dir-index.html @@ -25,12 +25,12 @@
-
About IPFS
-
Install IPFS
-
About
-
Install
+
About IPFS
+
Install IPFS
+
About
+
Install
@@ -83,7 +83,7 @@ {{ if .Hash }} - + {{ .ShortHash }} {{ end }} diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 5c9ebe80858..736ffc5ab34 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -12,6 +12,7 @@ import ( ipldlegacy "github.com/ipfs/go-ipld-legacy" ipath "github.com/ipfs/interface-go-ipfs-core/path" + dih "github.com/ipfs/kubo/assets/dag-index-html" "github.com/ipfs/kubo/tracing" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/multicodec" @@ -117,29 +118,16 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, } func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { - codecName := mc.Code(resolvedPath.Cid().Prefix().Codec).String() - body := fmt.Sprintf(` - - - - - -

Requested CID %q uses %q codec.

- - - -`, resolvedPath.Cid(), codecName) - - _, _ = w.Write([]byte(body)) + // TODO: cache-control/etag like for DirIndex + cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) + if err := dih.DagIndexTemplate.Execute(w, dih.DagIndexTemplateData{ + Path: contentPath.String(), + CID: resolvedPath.Cid().String(), + CodecName: cidCodec.String(), + CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + }); err != nil { + webError(w, "failed to generate HTML listing for this DAG: retry without 'Accept: text/html'", err, http.StatusInternalServerError) + } } func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string) { diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index f4dda631e43..0d2f07dbeaf 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -498,9 +498,9 @@ func TestIPNSHostnameBacklinks(t *testing.T) { if !strings.Contains(s, "") { t.Fatalf("expected file in directory listing") } - if !strings.Contains(s, "") { t.Fatalf("expected file in directory listing") } - if !strings.Contains(s, "" list_response +test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.tech" ' + test_should_contain "" list_response ' ## ============================================================================ From 7e848569ef7a17988dfaa4e8012a7ea6dfe32130 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 25 Nov 2022 23:25:41 +0100 Subject: [PATCH 28/32] fix(dag@gw): content type and cache headers This unifies the way we set headers for DAG- responses, adds missing ones for HTML response, and adds bunch of regression tests to ensure desired behaviors are not changed during future refactors. --- core/corehttp/gateway_handler_codec.go | 107 ++++++++---- test/sharness/t0116-gateway-cache.sh | 91 ++++++---- test/sharness/t0117-gateway-block.sh | 20 +-- test/sharness/t0123-gateway-json-cbor.sh | 204 +++++++++++++++++++++-- 4 files changed, 327 insertions(+), 95 deletions(-) diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go index 736ffc5ab34..95a151c7943 100644 --- a/core/corehttp/gateway_handler_codec.go +++ b/core/corehttp/gateway_handler_codec.go @@ -10,8 +10,10 @@ import ( "strings" "time" + cid "github.com/ipfs/go-cid" ipldlegacy "github.com/ipfs/go-ipld-legacy" ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipfs/kubo/assets" dih "github.com/ipfs/kubo/assets/dag-index-html" "github.com/ipfs/kubo/tracing" "github.com/ipld/go-ipld-prime" @@ -54,7 +56,12 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) defer span.End() - // If the resolved path still has some remainder, return bad request. + cidCodec := resolvedPath.Cid().Prefix().Codec + responseContentType := requestedContentType + + // If the resolved path still has some remainder, return error for now. + // TODO: handle this when we have IPLD Patch (https://ipld.io/specs/patch/) via HTTP PUT + // TODO: (depends on https://github.com/ipfs/kubo/issues/4801 and https://github.com/ipfs/kubo/issues/4782) if resolvedPath.Remainder() != "" { path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder()) err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", resolvedPath.Remainder(), resolvedPath.String(), path) @@ -62,25 +69,35 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } + // If no explicit content type was requested, the response will have one based on the codec from the CID + if requestedContentType == "" { + cidContentType, ok := codecToContentType[cidCodec] + if !ok { + // Should not happen unless function is called with wrong parameters. + err := fmt.Errorf("content type not found for codec: %v", cidCodec) + webError(w, "internal error", err, http.StatusInternalServerError) + return + } + responseContentType = cidContentType + } + + // Set HTTP headers (for caching etc) + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + name := setCodecContentDisposition(w, r, resolvedPath, responseContentType) + w.Header().Set("Content-Type", responseContentType) + w.Header().Set("X-Content-Type-Options", "nosniff") + // No content type is specified by the user (via Accept, or format=). However, // we support this format. Let's handle it. if requestedContentType == "" { - cidCodec := resolvedPath.Cid().Prefix().Codec isDAG := cidCodec == uint64(mc.DagJson) || cidCodec == uint64(mc.DagCbor) acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html") + download := r.URL.Query().Get("download") == "true" - if isDAG && acceptsHTML { + if isDAG && acceptsHTML && !download { i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath) } else { - cidContentType, ok := codecToContentType[cidCodec] - if !ok { - // Should not happen unless function is called with wrong parameters. - err := fmt.Errorf("content type not found for codec: %v", cidCodec) - webError(w, "internal error", err, http.StatusInternalServerError) - return - } - - i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, cidContentType) + i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime) } return @@ -96,10 +113,13 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, return } + // If we need to convert, use the last codec (strict dag- variant) + toCodec := codecs[len(codecs)-1] + // If the requested content type has "dag-", ALWAYS go through the encoding // process in order to validate the content. if strings.Contains(requestedContentType, "dag-") { - i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1]) + i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime) return } @@ -108,17 +128,32 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, // used here as it sets different headers. for _, codec := range codecs { if resolvedPath.Cid().Prefix().Codec == codec { - i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, requestedContentType) + i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime) return } } // Finally, if nothing of the above is true, we have to actually convert the codec. - i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, requestedContentType, codecs[len(codecs)-1]) + i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime) } func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { - // TODO: cache-control/etag like for DirIndex + // A HTML directory index will be presented, be sure to set the correct + // type instead of relying on autodetection (which may fail). + w.Header().Set("Content-Type", "text/html") + + // Clear Content-Disposition -- we want HTML to be rendered inline + w.Header().Del("Content-Disposition") + + // Generated index requires custom Etag (output may change between Kubo versions) + dagEtag := getDagIndexEtag(resolvedPath.Cid()) + w.Header().Set("Etag", dagEtag) + + // Remove Cache-Control for now to match UnixFS dir-index-html responses + // (we don't want browser to cache HTML forever) + // TODO: if we ever change behavior for UnixFS dir listings, same changes should be applied here + w.Header().Del("Cache-Control") + cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) if err := dih.DagIndexTemplate.Execute(w, dih.DagIndexTemplateData{ Path: contentPath.String(), @@ -126,16 +161,11 @@ func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWrit CodecName: cidCodec.String(), CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), }); err != nil { - webError(w, "failed to generate HTML listing for this DAG: retry without 'Accept: text/html'", err, http.StatusInternalServerError) + webError(w, "failed to generate HTML listing for this DAG: try fetching raw block with ?format=raw", err, http.StatusInternalServerError) } } -func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string) { - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) - name := setCodecContentDisposition(w, r, resolvedPath, contentType) - w.Header().Set("Content-Type", contentType) - w.Header().Set("X-Content-Type-Options", "nosniff") - +func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, name string, modtime time.Time) { blockCid := resolvedPath.Cid() blockReader, err := i.api.Block().Get(ctx, resolvedPath) if err != nil { @@ -154,7 +184,7 @@ func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWrite _, _, _ = ServeContent(w, r, name, modtime, content) } -func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, contentType string, codec uint64) { +func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec uint64, modtime time.Time) { obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid()) if err != nil { webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) @@ -169,14 +199,13 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons } finalNode := universal.(ipld.Node) - encoder, err := multicodec.LookupEncoder(codec) + encoder, err := multicodec.LookupEncoder(toCodec) if err != nil { webError(w, err.Error(), err, http.StatusInternalServerError) return } - // Keep it in memory so we can detect encoding errors in order to conform - // to the specification. + // Ensure IPLD node conforms to the codec specification. var buf bytes.Buffer err = encoder(finalNode, &buf) if err != nil { @@ -184,12 +213,6 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons return } - // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) - setCodecContentDisposition(w, r, resolvedPath, contentType) - w.Header().Set("Content-Type", contentType) - w.Header().Set("X-Content-Type-Options", "nosniff") - // Sets correct Last-Modified header. This code is borrowed from the standard // library (net/http/server.go) as we cannot use serveFile. if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { @@ -214,12 +237,22 @@ func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolved name = resolvedPath.Cid().String() + ext } - switch ext { - case ".json": // codecs that serialize to JSON can be rendered by browsers - dispType = "inline" - default: // everything else is assumed binary / opaque bytes + // JSON should be inlined, but ?download=true should still override + if r.URL.Query().Get("download") == "true" { dispType = "attachment" + } else { + switch ext { + case ".json": // codecs that serialize to JSON can be rendered by browsers + dispType = "inline" + default: // everything else is assumed binary / opaque bytes + dispType = "attachment" + } } + setContentDispositionHeader(w, name, dispType) return name } + +func getDagIndexEtag(dagCid cid.Cid) string { + return `"DagIndex-` + assets.AssetHash + `_CID-` + dagCid.String() + `"` +} diff --git a/test/sharness/t0116-gateway-cache.sh b/test/sharness/t0116-gateway-cache.sh index 2f13f93d896..0af4ec0eb71 100755 --- a/test/sharness/t0116-gateway-cache.sh +++ b/test/sharness/t0116-gateway-cache.sh @@ -16,10 +16,14 @@ test_launch_ipfs_daemon_without_network # - have implicit index.html for a good measure # /ipns/root1/root2/root3/ (/ipns/root1/root2/root3/index.html) -# Note: we cover important edge case here: +# Note: we cover important UnixFS-focused edge case here: +# # ROOT3_CID - dir listing (dir-index-html response) # ROOT4_CID - index.html returned as a root response (dir/), instead of generated dir-index-html # FILE_CID - index.html returned directly, as a file +# +# Caching of things like raw blocks, CARs, dag-json and dag-cbor +# is tested in their respective suites. test_expect_success "Add the test directory" ' mkdir -p root2/root3/root4 && @@ -41,30 +45,36 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' ' # GET /ipfs/ + # unixfs test_expect_success "GET for /ipfs/ unixfs dir listing succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/" >/dev/null 2>curl_ipfs_dir_listing_output && - cat curl_ipfs_dir_listing_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/" >/dev/null 2>curl_ipfs_dir_listing_output ' test_expect_success "GET for /ipfs/ unixfs dir with index.html succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/" >/dev/null 2>curl_ipfs_dir_index.html_output && - cat curl_ipfs_dir_index.html_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/" >/dev/null 2>curl_ipfs_dir_index.html_output ' test_expect_success "GET for /ipfs/ unixfs file succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/index.html" >/dev/null 2>curl_ipfs_file_output && - cat curl_ipfs_file_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/index.html" >/dev/null 2>curl_ipfs_file_output + ' + # unixfs dir as dag-json + test_expect_success "GET for /ipfs/ unixfs dir as DAG-JSON succeeds" ' + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/?format=dag-json" >/dev/null 2>curl_ipfs_dir_dag-json_output && + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/?format=json" >/dev/null 2>curl_ipfs_dir_json_output ' # GET /ipns/ + # unixfs test_expect_success "GET for /ipns/ unixfs dir listing succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/" >/dev/null 2>curl_ipns_dir_listing_output && - cat curl_ipns_dir_listing_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/" >/dev/null 2>curl_ipns_dir_listing_output ' test_expect_success "GET for /ipns/ unixfs dir with index.html succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/" >/dev/null 2>curl_ipns_dir_index.html_output && - cat curl_ipns_dir_index.html_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/" >/dev/null 2>curl_ipns_dir_index.html_output ' test_expect_success "GET for /ipns/ unixfs file succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" >/dev/null 2>curl_ipns_file_output && - cat curl_ipns_file_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" >/dev/null 2>curl_ipns_file_output + ' + # unixfs dir as dag-json + test_expect_success "GET for /ipns/ unixfs dir as DAG-JSON succeeds" ' + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/?format=dag-json" >/dev/null 2>curl_ipns_dir_dag-json_output && + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/?format=json" >/dev/null 2>curl_ipns_dir_json_output ' # Cache-Control @@ -82,19 +92,34 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' test_expect_success "GET /ipfs/ unixfs dir with index.html has expected Cache-Control" ' test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_index.html_output ' - +# Cache-Control: immutable /ipfs/ unixfs dir as dag-json + test_expect_success "GET /ipfs/ dag-json has expected Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_dag-json_output + ' +# Cache-Control: immutable /ipfs/ unixfs dir as json + test_expect_success "GET /ipfs/ unixfs dir as json has expected Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_json_output + ' # Cache-Control: mutable /ipns/ file test_expect_success "GET /ipns/ unixfs file has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_file_output ' -# Cache-Control: generated /ipns/dir/ (listing) +# Cache-Control: mutable /ipns/dir/ (generated listing) test_expect_success "GET /ipns/ unixfs dir listing has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_dir_listing_output ' -# Cache-Control: immutable /ipns/dir/ (index.html) +# Cache-Control: mutable /ipns/dir/ (index.html) test_expect_success "GET /ipns/ unixfs dir with index.html has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_dir_index.html_output ' +# Cache-Control: mutable /ipns/dir/ as dag-json + test_expect_success "GET /ipns/ unixfs dir as dag-json has no Cache-Control" ' + test_should_not_contain "< Cache-Control" curl_ipns_dir_dag-json_output + ' +# Cache-Control: mutable /ipns/dir/ as json + test_expect_success "GET /ipns/ unixfs dir as json has no Cache-Control" ' + test_should_not_contain "< Cache-Control" curl_ipns_dir_json_output + ' # Cache-Control: only-if-cached test_expect_success "HEAD for /ipfs/ with only-if-cached succeeds when in local datastore" ' @@ -122,78 +147,80 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' ## dir generated listing test_expect_success "GET /ipfs/ dir listing response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3" curl_ipfs_dir_listing_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir listing response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3" curl_ipns_dir_listing_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3" curl_ipns_dir_listing_output ' ## dir static index.html test_expect_success "GET /ipfs/ dir index.html response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/" curl_ipfs_dir_index.html_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/" curl_ipns_dir_index.html_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/" curl_ipns_dir_index.html_output ' # file test_expect_success "GET /ipfs/ file response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/index.html" curl_ipfs_file_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/index.html" curl_ipfs_file_output ' test_expect_success "GET /ipns/ file response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" curl_ipns_file_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" curl_ipns_file_output ' # X-Ipfs-Roots ## dir generated listing test_expect_success "GET /ipfs/ dir listing response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipfs_dir_listing_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir listing response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipns_dir_listing_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipns_dir_listing_output ' ## dir static index.html test_expect_success "GET /ipfs/ dir index.html response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipfs_dir_index.html_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipns_dir_index.html_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipns_dir_index.html_output ' ## file test_expect_success "GET /ipfs/ file response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipfs_file_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipfs_file_output ' test_expect_success "GET /ipns/ file response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipns_file_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipns_file_output ' # Etag ## dir generated listing test_expect_success "GET /ipfs/ dir response has special Etag for generated dir listing" ' + test_should_contain "< Etag: \"DirIndex" curl_ipfs_dir_listing_output && grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir response has special Etag for generated dir listing" ' + test_should_contain "< Etag: \"DirIndex" curl_ipfs_dir_listing_output && grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipns_dir_listing_output ' ## dir static index.html should use CID of the index.html file for improved HTTP caching test_expect_success "GET /ipfs/ dir index.html response has dir CID as Etag" ' - grep "< Etag: \"${ROOT4_CID}\"" curl_ipfs_dir_index.html_output + test_should_contain "< Etag: \"${ROOT4_CID}\"" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has dir CID as Etag" ' - grep "< Etag: \"${ROOT4_CID}\"" curl_ipns_dir_index.html_output + test_should_contain "< Etag: \"${ROOT4_CID}\"" curl_ipns_dir_index.html_output ' ## file test_expect_success "GET /ipfs/ response has CID as Etag for a file" ' - grep "< Etag: \"${FILE_CID}\"" curl_ipfs_file_output + test_should_contain "< Etag: \"${FILE_CID}\"" curl_ipfs_file_output ' test_expect_success "GET /ipns/ response has CID as Etag for a file" ' - grep "< Etag: \"${FILE_CID}\"" curl_ipns_file_output + test_should_contain "< Etag: \"${FILE_CID}\"" curl_ipns_file_output ' # If-None-Match (return 304 Not Modified when client sends matching Etag they already have) diff --git a/test/sharness/t0117-gateway-block.sh b/test/sharness/t0117-gateway-block.sh index 87228c1c54f..5f1d080a3d4 100755 --- a/test/sharness/t0117-gateway-block.sh +++ b/test/sharness/t0117-gateway-block.sh @@ -32,43 +32,41 @@ test_expect_success "Create text fixtures" ' test_expect_success "GET response for application/vnd.ipld.raw has expected Content-Type" ' curl -svX GET -H "Accept: application/vnd.ipld.raw" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir/ascii.txt" >/dev/null 2>curl_output && - cat curl_output && - grep "< Content-Type: application/vnd.ipld.raw" curl_output + test_should_contain "< Content-Type: application/vnd.ipld.raw" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Content-Length" ' BYTES=$(ipfs block get $FILE_CID | wc --bytes) - grep "< Content-Length: $BYTES" curl_output + test_should_contain "< Content-Length: $BYTES" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Content-Disposition" ' - grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.bin\"" curl_output + test_should_contain "< Content-Disposition: attachment\; filename=\"${FILE_CID}.bin\"" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes nosniff hint" ' - grep "< X-Content-Type-Options: nosniff" curl_output + test_should_contain "< X-Content-Type-Options: nosniff" curl_output ' test_expect_success "GET for application/vnd.ipld.raw with query filename includes Content-Disposition with custom filename" ' curl -svX GET -H "Accept: application/vnd.ipld.raw" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir/ascii.txt?filename=foobar.bin" >/dev/null 2>curl_output_filename && - cat curl_output_filename && - grep "< Content-Disposition: attachment\; filename=\"foobar.bin\"" curl_output_filename + test_should_contain "< Content-Disposition: attachment\; filename=\"foobar.bin\"" curl_output_filename ' # Cache control HTTP headers # (basic checks, detailed behavior is tested in t0116-gateway-cache.sh) test_expect_success "GET response for application/vnd.ipld.raw includes Etag" ' - grep "< Etag: \"${FILE_CID}.raw\"" curl_output + test_should_contain "< Etag: \"${FILE_CID}.raw\"" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes X-Ipfs-Path and X-Ipfs-Roots" ' - grep "< X-Ipfs-Path" curl_output && - grep "< X-Ipfs-Roots" curl_output + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Cache-Control" ' - grep "< Cache-Control: public, max-age=29030400, immutable" curl_output + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_output ' test_kill_ipfs_daemon diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh index 535249ab206..812d90f24b9 100755 --- a/test/sharness/t0123-gateway-json-cbor.sh +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -22,7 +22,9 @@ test_expect_success "Add the test directory" ' echo "$FILE_CID / $FILE_SIZE" ' -test_headers () { +## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON + +test_dag_pb_headers () { name=$1 format=$2 disposition=$3 @@ -56,8 +58,8 @@ test_headers () { ' } -test_headers "DAG-JSON" "json" "inline" -test_headers "DAG-CBOR" "cbor" "attachment" +test_dag_pb_headers "DAG-JSON" "json" "inline" +test_dag_pb_headers "DAG-CBOR" "cbor" "attachment" test_dag_pb () { name=$1 @@ -85,6 +87,8 @@ test_dag_pb () { test_dag_pb "DAG-JSON" "json" test_dag_pb "DAG-CBOR" "cbor" +## Content-Type response based on Accept header and ?format= parameter + test_cmp_dag_get () { name=$1 format=$2 @@ -136,6 +140,9 @@ test_cmp_dag_get () { test_cmp_dag_get "JSON" "json" "inline" test_cmp_dag_get "CBOR" "cbor" "attachment" + +## Lossless conversion between JSON and CBOR + test_expect_success "GET JSON as CBOR produces DAG-CBOR output" ' CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec json) && curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=cbor" > curl_output 2>&1 && @@ -150,6 +157,9 @@ test_expect_success "GET CBOR as JSON produces DAG-JSON output" ' test_cmp ipfs_dag_get_output curl_output ' + +## Pathing, traversal + DAG_CBOR_TRAVERSAL_CID="bafyreibs4utpgbn7uqegmd2goqz4bkyflre2ek2iwv743fhvylwi4zeeim" DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxkd47xq" DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke" @@ -163,12 +173,6 @@ test_expect_success "Add CARs for path traversal and DAG-PB representation tests test_should_contain $DAG_PB_CID import_output ' -test_expect_success "GET DAG-JSON with Accept: text/html returns HTML" ' - curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID" > curl_output 2>&1 && - test_should_not_contain "Content-Disposition" curl_output && - test_should_contain "Content-Type: text/html" curl_output -' - test_expect_success "GET DAG-JSON traversal returns 501 if there is path remainder" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo?format=dag-json" > curl_output 2>&1 && test_should_contain "501 Not Implemented" curl_output && @@ -182,12 +186,6 @@ test_expect_success "GET DAG-JSON traverses multiple links" ' test_cmp expected actual ' -test_expect_success "GET DAG-CBOR with Accept: text/html returns HTML" ' - curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID" > curl_output 2>&1 && - test_should_not_contain "Content-Disposition" curl_output && - test_should_contain "Content-Type: text/html" curl_output -' - test_expect_success "GET DAG-CBOR traversal returns 501 if there is path remainder" ' curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo?format=dag-cbor" > curl_output 2>&1 && test_should_contain "501 Not Implemented" curl_output && @@ -207,7 +205,183 @@ test_expect_success "GET DAG-CBOR traverses multiple links" ' # test_cmp ../t0123-gateway-json-cbor/dag-pb.json actual # ' + +## NATIVE TESTS: +## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested + + +test_native_dag () { + name=$1 + format=$2 + disposition=$3 + CID=$4 + + # GET without explicit format and Accept: text/html returns raw block + + test_expect_success "GET $name from /ipfs without explicit format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o curl_output && + test_cmp expected curl_output + ' + + # GET dag-cbor block via Accept and ?format and ensure both are the same as `ipfs block get` output + + test_expect_success "GET $name from /ipfs with format=dag-$format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o curl_ipfs_dag_param_output && + test_cmp expected curl_ipfs_dag_param_output + ' + + test_expect_success "GET $name from /ipfs with format=$format returns the same payload as format=dag-$format" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o curl_ipfs_dag_param_output && + test_cmp expected curl_ipfs_dag_param_output + ' + + test_expect_success "GET $name from /ipfs with application/vnd.ipld.dag-$format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected_block && + curl -sX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o curl_ipfs_dag_block_accept_output && + test_cmp expected_block curl_ipfs_dag_block_accept_output + ' + + # Make sure expected HTTP headers are returned with the dag- block + + test_expect_success "GET response for application/vnd.ipld.dag-$format has expected Content-Type" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" >/dev/null 2>curl_output && + test_should_contain "< Content-Type: application/vnd.ipld.dag-$format" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Content-Length" ' + BYTES=$(ipfs block get $CID | wc --bytes) + test_should_contain "< Content-Length: $BYTES" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Content-Disposition" ' + test_should_contain "< Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes nosniff hint" ' + test_should_contain "< X-Content-Type-Options: nosniff" curl_output + ' + + test_expect_success "GET for application/vnd.ipld.dag-$format with query filename includes Content-Disposition with custom filename" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?filename=foobar.$format" >/dev/null 2>curl_output_filename && + test_should_contain "< Content-Disposition: ${disposition}\; filename=\"foobar.$format\"" curl_output_filename + ' + + test_expect_success "GET for application/vnd.ipld.dag-$format with ?download=true forces Content-Disposition: attachment" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?filename=foobar.$format&download=true" >/dev/null 2>curl_output_filename && + test_should_contain "< Content-Disposition: attachment" curl_output_filename + ' + + # Cache control HTTP headers + # (basic checks, detailed behavior is tested in t0116-gateway-cache.sh) + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Etag" ' + test_should_contain "< Etag: \"${CID}.dag-$format\"" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes X-Ipfs-Path and X-Ipfs-Roots" ' + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_output + ' + + # HTTP HEAD behavior + test_expect_success "HEAD $name with no explicit format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD $name with an explicit JSON format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=json" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Etag: \"$CID.json\"" output && + test_should_contain "Content-Type: application/json" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD dag-pb with ?format=$format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Etag: \"$FILE_CID.$format\"" output && + test_should_contain "Content-Type: application/$format" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD $name with only-if-cached for missing block returns HTTP 412 Precondition Failed" ' + MISSING_CID=$(echo "{\"t\": \"$(date +%s)\"}" | ipfs dag put --store-codec=dag-${format}) && + ipfs block rm -f -q $MISSING_CID && + curl -I -H "Cache-Control: only-if-cached" "http://127.0.0.1:$GWAY_PORT/ipfs/$MISSING_CID" -o output && + test_should_contain "HTTP/1.1 412 Precondition Failed" output + ' + + # IPNS behavior (should be same as immutable /ipfs, but with different caching headers) + # To keep tests small we only confirm payload is the same, and then only test delta around caching headers. + + test_expect_success "Prepare IPNS with dag-$format" ' + IPNS_ID=$(ipfs key gen --ipns-base=base36 --type=ed25519 ${format}_test_key | head -n1 | tr -d "\n") && + ipfs name publish --key ${format}_test_key --allow-offline -Q "/ipfs/$CID" > name_publish_out && + test_check_peerid "${IPNS_ID}" && + ipfs name resolve "${IPNS_ID}" > output && + printf "/ipfs/%s\n" "$CID" > expected && + test_cmp expected output + ' + + test_expect_success "GET $name from /ipns without explicit format returns the same payload as /ipfs" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o ipfs_output && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" -o ipns_output && + test_cmp ipfs_output ipns_output + ' + + test_expect_success "GET $name from /ipns without explicit format returns the same payload as /ipfs" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o ipfs_output && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID?format=dag-$format" -o ipns_output && + test_cmp ipfs_output ipns_output + ' + + test_expect_success "GET $name from /ipns with explicit application/vnd.ipld.dag-$format has expected headers" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" >/dev/null 2>curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "< Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_contain "< Etag: \"${CID}.dag-$format\"" curl_output && + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output + ' + + + # When Accept header includes text/html and no explicit format is requested for DAG-(CBOR|JSON) + # The gateway returns generated HTML index (see dag-index-html) for web browsers (similar to dir-index-html returned for unixfs dirs) + # As this is generated, we don't return immutable Cache-Control, even on /ipfs (same as for dir-index-html) + + test_expect_success "GET $name on /ipfs with Accept: text/html returns HTML (dag-index-html)" ' + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + test_should_not_contain "Content-Disposition" curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "Etag: \"DagIndex-" curl_output && + test_should_contain "Content-Type: text/html" curl_output && + test_should_contain "" curl_output + ' + + test_expect_success "GET $name on /ipns with Accept: text/html returns HTML (dag-index-html)" ' + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" > curl_output 2>&1 && + test_should_not_contain "Content-Disposition" curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "Etag: \"DagIndex-" curl_output && + test_should_contain "Content-Type: text/html" curl_output && + test_should_contain "" curl_output + ' + + +} + +test_native_dag "DAG-JSON" "json" "inline" "$DAG_JSON_TRAVERSAL_CID" +test_native_dag "DAG-CBOR" "cbor" "attachment" "$DAG_CBOR_TRAVERSAL_CID" + test_kill_ipfs_daemon test_done +# vim: set ts=2 sw=2 et: From 162f4350b22ca69fc36a6a49f69e670ccfbbe3b9 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 28 Nov 2022 15:57:36 +0100 Subject: [PATCH 29/32] add changelog info --- docs/changelogs/v0.18.md | 73 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docs/changelogs/v0.18.md diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md new file mode 100644 index 00000000000..41d5a7b3b08 --- /dev/null +++ b/docs/changelogs/v0.18.md @@ -0,0 +1,73 @@ +# Kubo changelog v0.18 + +## v0.18.0 + +### Overview + +Below is an outline of all that is in this release, so you get a sense of all that's included. + +- [Kubo changelog v0.18](#kubo-changelog-v018) + - [v0.18.0](#v0180) + - [Overview](#overview) + - [🔦 Highlights](#-highlights) + - [TAR Response Format on Gateways](#tar-response-format-on-gateways) + - [Changelog](#changelog) + - [Contributors](#contributors) + +### 🔦 Highlights + +#### DAG-JSON and DAG-CBOR Response Formats on Gateways + +Implemented [IPIP-328](https://github.com/ipfs/specs/pull/328) which adds support +to DAG-JSON and DAG-CBOR, as well as their non-DAG variants, to the gateway. Now, +CIDs that encode JSON, CBOR, DAG-JSON and DAG-CBOR objects can be retrieved, and +traversed through IPLD Links. + +HTTP clients can request JSON, CBOR, DAG-JSON, and DAG-CBOR responses by either +passing the query parameter `?format` or setting the `Accept` HTTP header to the +following values: + +- JSON: `?format=json`, or `Accept: application/json` +- CBOR: `?format=cbor`, or `Accept: application/cbor` +- DAG-JSON: `?format=dag-json`, or `Accept: application/vnd.ipld.dag-json` +- DAG-JSON: `?format=dag-cbor`, or `Accept: application/vnd.ipld.dag-cbor` + +```console +$ export DIR_CID=bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq +$ curl -H "Accept: application/vnd.ipld.dag-json" "http://127.0.0.1:8080/ipfs/$DIR_CID" | jq +$ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq +{ + "Data": { + "/": { + "bytes": "CAE" + } + }, + "Links": [ + { + "Hash": { + "/": "Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi" + }, + "Name": "1 - Barrel - Part 1 - alt.txt", + "Tsize": 21 + }, + { + "Hash": { + "/": "QmdMxMx29KVYhHnaCc1icWYxQqXwUNCae6t1wS2NqruiHd" + }, + "Name": "1 - Barrel - Part 1 - transcript.txt", + "Tsize": 195 + }, + { + "Hash": { + "/": "QmawceGscqN4o8Y8Fv26UUmB454kn2bnkXV5tEQYc4jBd6" + }, + "Name": "1 - Barrel - Part 1.png", + "Tsize": 24862 + } + ] +} +``` + +### Changelog + +### Contributors From 12d0d7f36a642b799659e90981c0d200a624c48f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 28 Nov 2022 16:00:55 +0100 Subject: [PATCH 30/32] fix title --- docs/changelogs/v0.18.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md index 41d5a7b3b08..4afa6793ea9 100644 --- a/docs/changelogs/v0.18.md +++ b/docs/changelogs/v0.18.md @@ -10,13 +10,13 @@ Below is an outline of all that is in this release, so you get a sense of all th - [v0.18.0](#v0180) - [Overview](#overview) - [🔦 Highlights](#-highlights) - - [TAR Response Format on Gateways](#tar-response-format-on-gateways) + - [(DAG-)JSON and (DAG-)CBOR Response Formats on Gateways](#dag-json-and-dag-cbor-response-formats-on-gateways) - [Changelog](#changelog) - [Contributors](#contributors) ### 🔦 Highlights -#### DAG-JSON and DAG-CBOR Response Formats on Gateways +#### (DAG-)JSON and (DAG-)CBOR Response Formats on Gateways Implemented [IPIP-328](https://github.com/ipfs/specs/pull/328) which adds support to DAG-JSON and DAG-CBOR, as well as their non-DAG variants, to the gateway. Now, From 2d8ba784022bf0626a35d758567fad6dfa97d38d Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 30 Nov 2022 13:07:04 +0100 Subject: [PATCH 31/32] rm wild block --- ...m2zlpammeoi5llohicdvj5rvfwibwlinciwdbnixi.block | Bin 46 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 bafyreigcxa2x5mifam2zlpammeoi5llohicdvj5rvfwibwlinciwdbnixi.block diff --git a/bafyreigcxa2x5mifam2zlpammeoi5llohicdvj5rvfwibwlinciwdbnixi.block b/bafyreigcxa2x5mifam2zlpammeoi5llohicdvj5rvfwibwlinciwdbnixi.block deleted file mode 100644 index bc15f35a2bd5d2d57b32710d1eb5b5d40bb36967..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 46 zcmV+}0MY-UV`gt}*eX~h00C7JAY~j@qC<@3+j?UDKned8Y~_r;kC)7Y|Dxx_Ek++Y E=f}bood5s; From f084f094bf11e3a1e7a50d4d27b222d5efb9c95e Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 5 Dec 2022 17:41:22 +0100 Subject: [PATCH 32/32] fix(dag-index-html): remove technical jargon Applied feedback from IPFS Implementers Sync, HTML should be aimed at non-technical people, and UnixFS in bold is too intimidating. --- assets/dag-index-html/index.go | 1 - 1 file changed, 1 deletion(-) diff --git a/assets/dag-index-html/index.go b/assets/dag-index-html/index.go index dd1e93d799d..214b06a3822 100644 --- a/assets/dag-index-html/index.go +++ b/assets/dag-index-html/index.go @@ -48,7 +48,6 @@ var DagIndexTemplate = template.Must(template.New("redirect").Parse(`

CID: {{.CID}}
Codec: {{.CodecName}} ({{.CodecHex}})

-

This CID is not UnixFS (not a file, nor a directory)