diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index e1b57c180..15e00383d 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.9 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10 with: output: fixtures merged: true @@ -47,7 +47,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.9 + uses: ipfs/gateway-conformance/.github/actions/test@v0.10 with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +84,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.9 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10 with: output: fixtures merged: true @@ -114,7 +114,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.9 + uses: ipfs/gateway-conformance/.github/actions/test@v0.10 with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +152,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.9 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10 with: output: fixtures merged: true @@ -182,7 +182,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.9 + uses: ipfs/gateway-conformance/.github/actions/test@v0.10 with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f38e6a42..b12ac141f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,9 @@ The following emojis are used to highlight certain changes: ### Changed - 🛠 `chunker`: `DefaultBlockSize` changed from `const` to `var` to allow runtime configuration via global profiles. [#1088](https://github.com/ipfs/boxo/pull/1088), [IPIP-499](https://github.com/ipfs/specs/pull/499) -- `gateway`: ✨ [IPIP-523](https://github.com/ipfs/specs/pull/523) `?format=` URL query parameter now takes precedence over `Accept` HTTP header, ensuring deterministic HTTP cache behavior and allowing browsers to use `?format=` even when they send `Accept` headers with specific content types. [#1074](https://github.com/ipfs/boxo/pull/1074) +- `gateway`: 🛠 ✨ [IPIP-523](https://github.com/ipfs/specs/pull/523) `?format=` URL query parameter now takes precedence over `Accept` HTTP header, ensuring deterministic HTTP cache behavior and allowing browsers to use `?format=` even when they send `Accept` headers with specific content types. [#1074](https://github.com/ipfs/boxo/pull/1074) +- `gateway`: 🛠 ✨ [IPIP-524](https://github.com/ipfs/specs/pull/524) codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default. Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable with a hint to fetch raw blocks (`?format=raw`) and convert client-side. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. [#1077](https://github.com/ipfs/boxo/pull/1077) +- `gateway`: compliance with gateway-conformance [v0.10.0](https://github.com/ipfs/gateway-conformance/releases/tag/v0.10.0) (since v0.8: relaxed DAG-CBOR HTML preview cache headers, relaxed CAR 200/404 for missing paths, [IPIP-523](https://github.com/ipfs/specs/pull/523) format query precedence, [IPIP-524](https://github.com/ipfs/specs/pull/524) codec mismatch returns 406) ### Removed diff --git a/examples/gateway/car-file/main_test.go b/examples/gateway/car-file/main_test.go index 6c8fc22d7..6e611d819 100644 --- a/examples/gateway/car-file/main_test.go +++ b/examples/gateway/car-file/main_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "io" "net/http" "net/http/httptest" @@ -8,8 +9,6 @@ import ( "github.com/ipfs/boxo/examples/gateway/common" "github.com/ipfs/boxo/gateway" - "github.com/ipld/go-ipld-prime/codec/dagjson" - "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/stretchr/testify/assert" ) @@ -62,48 +61,35 @@ func TestFile(t *testing.T) { assert.EqualValues(t, string(body), "hello world\n") } -func TestDirectoryAsDAG(t *testing.T) { +func TestDirectoryAsRawBlock(t *testing.T) { ts, f, err := newTestServer() assert.NoError(t, err) defer f.Close() - res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=dag-json") + res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=raw") assert.NoError(t, err) defer res.Body.Close() - contentType := res.Header.Get("Content-Type") - assert.EqualValues(t, contentType, "application/vnd.ipld.dag-json") - - // Parses the DAG-JSON response. - dag := basicnode.Prototype.Any.NewBuilder() - err = dagjson.Decode(dag, res.Body) - assert.NoError(t, err) - - // Checks for the links inside the logical model. - links, err := dag.Build().LookupByString("Links") - assert.NoError(t, err) - - // Checks if there are 2 links. - assert.EqualValues(t, links.Length(), 2) - - // Check if the first item is correct. - n, err := links.LookupByIndex(0) - assert.NoError(t, err) - assert.NotNil(t, n) + assert.Equal(t, http.StatusOK, res.StatusCode) - nameNode, err := n.LookupByString("Name") - assert.NoError(t, err) - assert.NotNil(t, nameNode) - - name, err := nameNode.AsString() - assert.NoError(t, err) - assert.EqualValues(t, name, "eye.png") + contentType := res.Header.Get("Content-Type") + assert.Equal(t, "application/vnd.ipld.raw", contentType) - hashNode, err := n.LookupByString("Hash") + body, err := io.ReadAll(res.Body) assert.NoError(t, err) - assert.NotNil(t, hashNode) - hash, err := hashNode.AsLink() - assert.NoError(t, err) - assert.EqualValues(t, hash.String(), "bafybeigmlfksb374fdkxih4urny2yiyazyra2375y2e4a72b3jcrnthnau") + // Raw bytes of the dag-pb directory block + expected := []byte{ + 0x12, 0x33, 0x0a, 0x24, 0x01, 0x70, 0x12, 0x20, 0xcc, 0x59, 0x55, 0x20, + 0xef, 0xfc, 0x28, 0xd5, 0x74, 0x1f, 0x94, 0x8b, 0x71, 0xac, 0x23, 0x00, + 0xce, 0x22, 0x0d, 0x6f, 0xfd, 0xc6, 0x89, 0xc0, 0x7f, 0x41, 0xda, 0x45, + 0x16, 0xcc, 0xed, 0x05, 0x12, 0x07, 0x65, 0x79, 0x65, 0x2e, 0x70, 0x6e, + 0x67, 0x18, 0xd0, 0xc8, 0x10, 0x12, 0x33, 0x0a, 0x24, 0x01, 0x55, 0x12, + 0x20, 0xa9, 0x48, 0x90, 0x4f, 0x2f, 0x0f, 0x47, 0x9b, 0x8f, 0x81, 0x97, + 0x69, 0x4b, 0x30, 0x18, 0x4b, 0x0d, 0x2e, 0xd1, 0xc1, 0xcd, 0x2a, 0x1e, + 0xc0, 0xfb, 0x85, 0xd2, 0x99, 0xa1, 0x92, 0xa4, 0x47, 0x12, 0x09, 0x68, + 0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x74, 0x78, 0x74, 0x18, 0x0c, 0x0a, 0x02, + 0x08, 0x01, + } + assert.True(t, bytes.Equal(body, expected), "raw block bytes should match") } diff --git a/examples/go.mod b/examples/go.mod index 5f41c1e06..86e447de6 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -8,7 +8,6 @@ require ( github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-datastore v0.9.0 github.com/ipld/go-car/v2 v2.16.0 - github.com/ipld/go-ipld-prime v0.21.0 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multicodec v0.10.0 @@ -63,6 +62,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.3 // indirect github.com/ipfs/go-unixfsnode v1.10.2 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect + github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect diff --git a/gateway/assets/assets.go b/gateway/assets/assets.go index b541d21ba..d123cc6cf 100644 --- a/gateway/assets/assets.go +++ b/gateway/assets/assets.go @@ -96,11 +96,12 @@ type GlobalData struct { type DagTemplateData struct { GlobalData - Path string - CID string - CodecName string - CodecHex string - Node *ParsedNode + Path string + CID string + CodecName string + CodecHex string + Node *ParsedNode + AllowCodecConversion bool } type ErrorTemplateData struct { diff --git a/gateway/assets/dag.html b/gateway/assets/dag.html index 61d799a5b..ca681194b 100644 --- a/gateway/assets/dag.html +++ b/gateway/assets/dag.html @@ -23,9 +23,9 @@

You can download this block as:

{{ with .Node }} diff --git a/gateway/gateway.go b/gateway/gateway.go index ce6d4b49f..a22e3e4e9 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -56,6 +56,19 @@ type Config struct { // [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/ DeserializedResponses bool + // AllowCodecConversion enables automatic conversion between codecs when + // the requested format differs from the block's native codec. For example, + // converting dag-pb (UnixFS) to dag-json. + // + // When false (default), the gateway returns 406 Not Acceptable if the + // requested format doesn't match the block's codec. This follows the + // behavior specified in IPIP-524. + // + // When true, the gateway attempts to convert between legacy IPLD formats. + // This is provided for backwards compatibility but is not required by + // the gateway specification. + AllowCodecConversion bool + // NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in // response to requests with values in `Host` HTTP header. This flag can be // overridden per FQDN in PublicGateways. To be used with WithHostname. diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 123cff6b1..476aca9e2 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -522,6 +522,7 @@ func TestHeaders(t *testing.T) { }, }, DeserializedResponses: true, + AllowCodecConversion: true, // Test tests various format conversions }) runTest := func(name, path, accept, host, expectedContentLocationHdr string) { @@ -1097,7 +1098,8 @@ func TestDeserializedResponses(t *testing.T) { backend, root := newMockBackend(t, "fixtures.car") ts := newTestServerWithConfig(t, backend, Config{ - NoDNSLink: false, + NoDNSLink: false, + AllowCodecConversion: true, // Test expects codec conversions to work PublicGateways: map[string]*PublicGateway{ "trustless.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -1176,7 +1178,8 @@ func TestDeserializedResponses(t *testing.T) { backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ - NoDNSLink: false, + NoDNSLink: false, + AllowCodecConversion: true, // Test expects codec conversions to work PublicGateways: map[string]*PublicGateway{ "trustless.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -1210,6 +1213,110 @@ func TestDeserializedResponses(t *testing.T) { }) } +func TestAllowCodecConversion(t *testing.T) { + t.Parallel() + + cborBackend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car") + pbBackend, dagPbRoot := newMockBackend(t, "fixtures.car") + + // Positive cases: matching codec or conversion enabled should return 200 + t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) { + t.Parallel() + ts := newTestServerWithConfig(t, cborBackend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) + }) + + t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) { + t.Parallel() + ts := newTestServerWithConfig(t, cborBackend, Config{ + DeserializedResponses: true, + AllowCodecConversion: true, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) + }) + + // Negative cases: requesting dag-json or dag-cbor for a block with a + // different codec should return 406 with an actionable error hint. + for _, tc := range []struct { + name string + backend IPFSBackend + path string + format string + }{ + {"dag-cbor block with dag-json", cborBackend, "/ipfs/" + dagCborRoot.String(), "dag-json"}, + {"dag-pb directory with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/", "dag-json"}, + {"dag-pb directory with dag-cbor", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/", "dag-cbor"}, + {"dag-pb file with dag-json", pbBackend, "/ipfs/bafyaacqkbaeaeeqcpn6rqaq", "dag-json"}, + {"raw block with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/fnord", "dag-json"}, + } { + t.Run("AllowCodecConversion=false returns 406 for "+tc.name, func(t *testing.T) { + t.Parallel() + ts := newTestServerWithConfig(t, tc.backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+tc.path+"?format="+tc.format, nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + assert.Contains(t, string(body), errCodecConversionHint) + }) + } + + // Ensure ?format=json on dag-pb and raw content serves the default + // response (not 406). JSON files may be stored as dag-pb or raw (UnixFS) + // and ?format=json goes through serveDefaults which does not do codec + // conversion - it serves directory listings or file bytes as-is. + for _, tc := range []struct { + name string + path string + }{ + {"dag-pb directory", "/ipfs/" + dagPbRoot.String() + "/subdir/"}, + {"dag-pb file", "/ipfs/" + dagPbRoot.String() + "/subdir/fnord"}, + } { + t.Run("AllowCodecConversion=false serves default response for "+tc.name+" with json format", func(t *testing.T) { + t.Parallel() + ts := newTestServerWithConfig(t, pbBackend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+tc.path+"?format=json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.NotEqual(t, http.StatusNotAcceptable, res.StatusCode) + }) + } + + // Ensure dag-pb file with Accept: application/json is not rejected. + // This guards behavior where JSON files are stored as dag-pb or raw + // (UnixFS) and requested by clients with Accept: application/json. + // Regular HTTP use must not be broken by overreaching HTTP 406 from IPIP-524. + t.Run("AllowCodecConversion=false allows dag-pb file with Accept application/json", func(t *testing.T) { + t.Parallel() + ts := newTestServerWithConfig(t, pbBackend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagPbRoot.String()+"/subdir/fnord", nil) + req.Header.Set("Accept", "application/json") + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.NotEqual(t, http.StatusNotAcceptable, res.StatusCode) + }) +} + type errorMockBackend struct { err error } diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 49b2e9952..414c89900 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -57,6 +57,10 @@ var contentTypeToExtension = map[string]string{ dagCborResponseFormat: ".cbor", } +// errCodecConversionHint is the user-facing hint returned in 406 responses +// when codec conversion is not allowed (IPIP-524). +const errCodecConversionHint = "codec conversion is not supported, fetch raw block with ?format=raw and convert client-side" + func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { ctx, span := spanTrace(ctx, "Handler.ServeCodec", trace.WithAttributes(attribute.String("path", rq.immutablePath.String()), attribute.String("requestedContentType", rq.responseFormat))) defer span.End() @@ -148,7 +152,20 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return false } - // This handles DAG-* conversions and validations. + // IPIP-524: Check if codec conversion is allowed + if !i.config.AllowCodecConversion && toCodec != cidCodec { + // Conversion not allowed and codecs don't match - return 406 + err := fmt.Errorf("format %q requested but block has codec %q: %s", rq.responseFormat, cidCodec.String(), errCodecConversionHint) + i.webError(w, r, err, http.StatusNotAcceptable) + return false + } + + // If codecs match, serve raw (no conversion needed) + if toCodec == cidCodec { + return i.serveCodecRaw(ctx, w, r, blockSize, blockData, rq.contentPath, modtime, rq.begin) + } + + // AllowCodecConversion is true - perform DAG-* conversion return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin) } @@ -197,12 +214,13 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * cidCodec := mc.Code(resolvedPath.RootCid().Prefix().Codec) err = assets.DagTemplate.Execute(w, assets.DagTemplateData{ - GlobalData: i.getTemplateGlobalData(r, contentPath), - Path: contentPath.String(), - CID: resolvedPath.RootCid().String(), - CodecName: cidCodec.String(), - CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), - Node: parseNode(blockCid, blockData), + GlobalData: i.getTemplateGlobalData(r, contentPath), + Path: contentPath.String(), + CID: resolvedPath.RootCid().String(), + CodecName: cidCodec.String(), + CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + Node: parseNode(blockCid, blockData), + AllowCodecConversion: i.config.AllowCodecConversion, }) if err != nil { _, _ = fmt.Fprintf(w, "error during body generation: %v", err) diff --git a/gateway/handler_codec_test.go b/gateway/handler_codec_test.go index 127e0bc8c..90084f48b 100644 --- a/gateway/handler_codec_test.go +++ b/gateway/handler_codec_test.go @@ -79,4 +79,48 @@ func TestDagJsonCborPreview(t *testing.T) { require.Contains(t, string(body), escaped) require.NotContains(t, string(body), script) }) + + t.Run("download links without AllowCodecConversion", func(t *testing.T) { + t.Parallel() + + tsNoConv := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + + req := mustNewRequest(t, http.MethodGet, tsNoConv.URL+resolvedPath.String()+"/", nil) + req.Header.Add("Accept", "text/html") + + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusOK, res.StatusCode) + + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + + require.Contains(t, string(body), `href="?format=raw"`, "raw block link always present") + require.Contains(t, string(body), `href="?format=dag-cbor"`, "native codec link present") + require.NotContains(t, string(body), `href="?format=dag-json"`, "cross-codec link absent when conversion disabled") + }) + + t.Run("download links with AllowCodecConversion", func(t *testing.T) { + t.Parallel() + + tsConv := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: true, + }) + + req := mustNewRequest(t, http.MethodGet, tsConv.URL+resolvedPath.String()+"/", nil) + req.Header.Add("Accept", "text/html") + + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusOK, res.StatusCode) + + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + + require.Contains(t, string(body), `href="?format=raw"`, "raw block link always present") + require.Contains(t, string(body), `href="?format=dag-cbor"`, "native codec link present") + require.Contains(t, string(body), `href="?format=dag-json"`, "cross-codec link present when conversion enabled") + }) } diff --git a/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car b/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car new file mode 100644 index 000000000..92c3d4f3e Binary files /dev/null and b/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car differ diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index c186455ea..b7743ef7a 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -234,6 +234,7 @@ func newTestServerAndNode(t *testing.T, fixturesFile string) (*httptest.Server, func newTestServer(t *testing.T, backend IPFSBackend) *httptest.Server { return newTestServerWithConfig(t, backend, Config{ DeserializedResponses: true, + AllowCodecConversion: true, // Enable for backwards compatibility in tests MetricsRegistry: prometheus.NewRegistry(), }) }