From c92cc346db4695321b4f90261f20e8956eba30ad Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 3 Dec 2025 03:59:46 +0100 Subject: [PATCH 1/6] feat(gateway): add AllowCodecConversion config option Add AllowCodecConversion to gateway.Config to control codec conversion behavior per IPIP-0524. When false (default), the gateway returns 406 Not Acceptable if the requested format doesn't match the block's codec. When true, conversions between codecs are performed for backward compatibility. Codec conversion tests moved here from gateway-conformance since conversions are now an optional implementation feature, not a spec requirement. Gateway-conformance now tests for 406 responses. Ref: https://github.com/ipfs/specs/pull/524 Ref: https://github.com/ipfs/gateway-conformance/pull/254 --- .github/workflows/gateway-conformance.yml | 12 ++-- CHANGELOG.md | 2 + examples/gateway/car-file/main_test.go | 56 +++++++---------- examples/go.mod | 2 +- gateway/gateway.go | 13 ++++ gateway/gateway_test.go | 59 +++++++++++++++++- gateway/handler_codec.go | 15 ++++- .../path_gateway_dag/dag-cbor-traversal.car | Bin 0 -> 318 bytes gateway/utilities_test.go | 1 + 9 files changed, 115 insertions(+), 45 deletions(-) create mode 100644 gateway/testdata/path_gateway_dag/dag-cbor-traversal.car diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 47bb861e7..6a8d97de1 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -47,7 +47,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +84,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -114,7 +114,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +152,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -182,7 +182,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b6576431..56b9a6712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ The following emojis are used to highlight certain changes: ### Changed +- `gateway`: 🛠 Codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default per [IPIP-0524](https://github.com/ipfs/specs/pull/524). Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable. Clients should fetch raw blocks (`?format=raw`) and convert in userland. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. + ### Removed ### Fixed diff --git a/examples/gateway/car-file/main_test.go b/examples/gateway/car-file/main_test.go index 6c8fc22d7..6e611d819 100644 --- a/examples/gateway/car-file/main_test.go +++ b/examples/gateway/car-file/main_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "io" "net/http" "net/http/httptest" @@ -8,8 +9,6 @@ import ( "github.com/ipfs/boxo/examples/gateway/common" "github.com/ipfs/boxo/gateway" - "github.com/ipld/go-ipld-prime/codec/dagjson" - "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/stretchr/testify/assert" ) @@ -62,48 +61,35 @@ func TestFile(t *testing.T) { assert.EqualValues(t, string(body), "hello world\n") } -func TestDirectoryAsDAG(t *testing.T) { +func TestDirectoryAsRawBlock(t *testing.T) { ts, f, err := newTestServer() assert.NoError(t, err) defer f.Close() - res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=dag-json") + res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=raw") assert.NoError(t, err) defer res.Body.Close() - contentType := res.Header.Get("Content-Type") - assert.EqualValues(t, contentType, "application/vnd.ipld.dag-json") - - // Parses the DAG-JSON response. - dag := basicnode.Prototype.Any.NewBuilder() - err = dagjson.Decode(dag, res.Body) - assert.NoError(t, err) - - // Checks for the links inside the logical model. - links, err := dag.Build().LookupByString("Links") - assert.NoError(t, err) - - // Checks if there are 2 links. - assert.EqualValues(t, links.Length(), 2) - - // Check if the first item is correct. - n, err := links.LookupByIndex(0) - assert.NoError(t, err) - assert.NotNil(t, n) + assert.Equal(t, http.StatusOK, res.StatusCode) - nameNode, err := n.LookupByString("Name") - assert.NoError(t, err) - assert.NotNil(t, nameNode) - - name, err := nameNode.AsString() - assert.NoError(t, err) - assert.EqualValues(t, name, "eye.png") + contentType := res.Header.Get("Content-Type") + assert.Equal(t, "application/vnd.ipld.raw", contentType) - hashNode, err := n.LookupByString("Hash") + body, err := io.ReadAll(res.Body) assert.NoError(t, err) - assert.NotNil(t, hashNode) - hash, err := hashNode.AsLink() - assert.NoError(t, err) - assert.EqualValues(t, hash.String(), "bafybeigmlfksb374fdkxih4urny2yiyazyra2375y2e4a72b3jcrnthnau") + // Raw bytes of the dag-pb directory block + expected := []byte{ + 0x12, 0x33, 0x0a, 0x24, 0x01, 0x70, 0x12, 0x20, 0xcc, 0x59, 0x55, 0x20, + 0xef, 0xfc, 0x28, 0xd5, 0x74, 0x1f, 0x94, 0x8b, 0x71, 0xac, 0x23, 0x00, + 0xce, 0x22, 0x0d, 0x6f, 0xfd, 0xc6, 0x89, 0xc0, 0x7f, 0x41, 0xda, 0x45, + 0x16, 0xcc, 0xed, 0x05, 0x12, 0x07, 0x65, 0x79, 0x65, 0x2e, 0x70, 0x6e, + 0x67, 0x18, 0xd0, 0xc8, 0x10, 0x12, 0x33, 0x0a, 0x24, 0x01, 0x55, 0x12, + 0x20, 0xa9, 0x48, 0x90, 0x4f, 0x2f, 0x0f, 0x47, 0x9b, 0x8f, 0x81, 0x97, + 0x69, 0x4b, 0x30, 0x18, 0x4b, 0x0d, 0x2e, 0xd1, 0xc1, 0xcd, 0x2a, 0x1e, + 0xc0, 0xfb, 0x85, 0xd2, 0x99, 0xa1, 0x92, 0xa4, 0x47, 0x12, 0x09, 0x68, + 0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x74, 0x78, 0x74, 0x18, 0x0c, 0x0a, 0x02, + 0x08, 0x01, + } + assert.True(t, bytes.Equal(body, expected), "raw block bytes should match") } diff --git a/examples/go.mod b/examples/go.mod index ebdaacd90..92112616e 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -8,7 +8,6 @@ require ( github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-datastore v0.9.0 github.com/ipld/go-car/v2 v2.16.0 - github.com/ipld/go-ipld-prime v0.21.0 github.com/libp2p/go-libp2p v0.45.0 github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multicodec v0.10.0 @@ -63,6 +62,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.2 // indirect github.com/ipfs/go-unixfsnode v1.10.2 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect + github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect diff --git a/gateway/gateway.go b/gateway/gateway.go index 8cec10f21..cd78f8280 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -52,6 +52,19 @@ type Config struct { // [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/ DeserializedResponses bool + // AllowCodecConversion enables automatic conversion between codecs when + // the requested format differs from the block's native codec. For example, + // converting dag-pb (UnixFS) to dag-json. + // + // When false (default), the gateway returns 406 Not Acceptable if the + // requested format doesn't match the block's codec. This follows the + // behavior specified in IPIP-0524. + // + // When true, the gateway attempts to convert between legacy IPLD formats. + // This is provided for backwards compatibility but is not required by + // the gateway specification. + AllowCodecConversion bool + // NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in // response to requests with values in `Host` HTTP header. This flag can be // overridden per FQDN in PublicGateways. To be used with WithHostname. diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index b0fc7fa7d..5f37fdb05 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -521,6 +521,7 @@ func TestHeaders(t *testing.T) { }, }, DeserializedResponses: true, + AllowCodecConversion: true, // Test tests various format conversions }) runTest := func(name, path, accept, host, expectedContentLocationHdr string) { @@ -1073,7 +1074,8 @@ func TestDeserializedResponses(t *testing.T) { backend, root := newMockBackend(t, "fixtures.car") ts := newTestServerWithConfig(t, backend, Config{ - NoDNSLink: false, + NoDNSLink: false, + AllowCodecConversion: true, // Test expects codec conversions to work PublicGateways: map[string]*PublicGateway{ "trustless.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -1152,7 +1154,8 @@ func TestDeserializedResponses(t *testing.T) { backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ - NoDNSLink: false, + NoDNSLink: false, + AllowCodecConversion: true, // Test expects codec conversions to work PublicGateways: map[string]*PublicGateway{ "trustless.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -1186,6 +1189,58 @@ func TestDeserializedResponses(t *testing.T) { }) } +func TestAllowCodecConversion(t *testing.T) { + t.Parallel() + + // Use dag-cbor fixture + backend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car") + + t.Run("AllowCodecConversion=false returns 406 for codec mismatch", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, // IPIP-0524 behavior + }) + + // Request dag-json for a dag-cbor block - should return 406 + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + }) + + t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, // IPIP-0524 behavior + }) + + // Request dag-cbor for a dag-cbor block - should return 200 + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) + }) + + t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: true, // Legacy behavior + }) + + // Request dag-json for a dag-cbor block - should return 200 with conversion + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) + }) +} + type errorMockBackend struct { err error } diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 49b2e9952..122756423 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -148,7 +148,20 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return false } - // This handles DAG-* conversions and validations. + // IPIP-0524: Check if codec conversion is allowed + if !i.config.AllowCodecConversion && toCodec != cidCodec { + // Conversion not allowed and codecs don't match - return 406 + err := fmt.Errorf("format %q requested but block has codec %q: codec conversion is not supported", rq.responseFormat, cidCodec.String()) + i.webError(w, r, err, http.StatusNotAcceptable) + return false + } + + // If codecs match, serve raw (no conversion needed) + if toCodec == cidCodec { + return i.serveCodecRaw(ctx, w, r, blockSize, blockData, rq.contentPath, modtime, rq.begin) + } + + // AllowCodecConversion is true - perform DAG-* conversion return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin) } diff --git a/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car b/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car new file mode 100644 index 0000000000000000000000000000000000000000..92c3d4f3e21718627007431c6b6d0cc1693be206 GIT binary patch literal 318 zcmcColvlXV(2oIMVptN{kPzW{vS&r0ub_{_K>mhu-w1 zv^b}ir4|)u=I1dM5Tklwa$0`=qLiG>yll7)cbQITWL7ctu-q3j&R(lE-F}^dl&{i> zKYDy?e%dYj{535 Date: Thu, 5 Feb 2026 19:45:11 +0100 Subject: [PATCH 2/6] ci: update gateway-conformance to e17586f4 --- .github/workflows/gateway-conformance.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 6a8d97de1..4fabae220 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -47,7 +47,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/test@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +84,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -114,7 +114,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/test@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +152,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -182,7 +182,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/test@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 From fe168324d0667fabd4d892b01196a94214566bce Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 5 Feb 2026 21:44:05 +0100 Subject: [PATCH 3/6] fix(gateway): 406 error now tells you how to get the data you need when codec conversion is disabled (IPIP-0524) and you request ?format=dag-json for a dag-pb block, the 406 response now suggests fetching the raw block with ?format=raw and converting client-side. covers dag-pb directories, dag-pb files, and raw blocks requested with ?format=dag-json or ?format=dag-cbor. plain ?format=json and Accept: application/json continue to serve the default response, so existing HTTP clients are not affected. --- gateway/gateway_test.go | 102 +++++++++++++++++++++++++++++---------- gateway/handler_codec.go | 6 ++- 2 files changed, 82 insertions(+), 26 deletions(-) diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 3e2bf66bd..29728e91d 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -1216,52 +1216,104 @@ func TestDeserializedResponses(t *testing.T) { func TestAllowCodecConversion(t *testing.T) { t.Parallel() - // Use dag-cbor fixture - backend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car") + cborBackend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car") + pbBackend, dagPbRoot := newMockBackend(t, "fixtures.car") - t.Run("AllowCodecConversion=false returns 406 for codec mismatch", func(t *testing.T) { + // Positive cases: matching codec or conversion enabled should return 200 + t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) { t.Parallel() - - ts := newTestServerWithConfig(t, backend, Config{ + ts := newTestServerWithConfig(t, cborBackend, Config{ DeserializedResponses: true, - AllowCodecConversion: false, // IPIP-0524 behavior + AllowCodecConversion: false, }) - - // Request dag-json for a dag-cbor block - should return 406 - req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil) res := mustDoWithoutRedirect(t, req) defer res.Body.Close() - assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + assert.Equal(t, http.StatusOK, res.StatusCode) }) - t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) { + t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) { t.Parallel() - - ts := newTestServerWithConfig(t, backend, Config{ + ts := newTestServerWithConfig(t, cborBackend, Config{ DeserializedResponses: true, - AllowCodecConversion: false, // IPIP-0524 behavior + AllowCodecConversion: true, }) - - // Request dag-cbor for a dag-cbor block - should return 200 - req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) res := mustDoWithoutRedirect(t, req) defer res.Body.Close() assert.Equal(t, http.StatusOK, res.StatusCode) }) - t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) { - t.Parallel() + // Negative cases: requesting dag-json or dag-cbor for a block with a + // different codec should return 406 with an actionable error hint. + for _, tc := range []struct { + name string + backend IPFSBackend + path string + format string + }{ + {"dag-cbor block with dag-json", cborBackend, "/ipfs/" + dagCborRoot.String(), "dag-json"}, + {"dag-pb directory with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/", "dag-json"}, + {"dag-pb directory with dag-cbor", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/", "dag-cbor"}, + {"dag-pb file with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/fnord", "dag-json"}, + {"raw block with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/fnord", "dag-json"}, + } { + t.Run("AllowCodecConversion=false returns 406 for "+tc.name, func(t *testing.T) { + t.Parallel() + ts := newTestServerWithConfig(t, tc.backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+tc.path+"?format="+tc.format, nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() - ts := newTestServerWithConfig(t, backend, Config{ - DeserializedResponses: true, - AllowCodecConversion: true, // Legacy behavior + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + assert.Contains(t, string(body), errCodecConversionHint) }) + } - // Request dag-json for a dag-cbor block - should return 200 with conversion - req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + // Ensure ?format=json on dag-pb and raw content serves the default + // response (not 406). JSON files may be stored as dag-pb or raw (UnixFS) + // and ?format=json goes through serveDefaults which does not do codec + // conversion - it serves directory listings or file bytes as-is. + for _, tc := range []struct { + name string + path string + }{ + {"dag-pb directory", "/ipfs/" + dagPbRoot.String() + "/subdir/"}, + {"dag-pb file", "/ipfs/" + dagPbRoot.String() + "/subdir/fnord"}, + } { + t.Run("AllowCodecConversion=false serves default response for "+tc.name+" with json format", func(t *testing.T) { + t.Parallel() + ts := newTestServerWithConfig(t, pbBackend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+tc.path+"?format=json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.NotEqual(t, http.StatusNotAcceptable, res.StatusCode) + }) + } + + // Ensure dag-pb file with Accept: application/json is not rejected. + // This guards behavior where JSON files are stored as dag-pb or raw + // (UnixFS) and requested by clients with Accept: application/json. + // Regular HTTP use must not be broken by overreaching HTTP 406 from IPIP-524. + t.Run("AllowCodecConversion=false allows dag-pb file with Accept application/json", func(t *testing.T) { + t.Parallel() + ts := newTestServerWithConfig(t, pbBackend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagPbRoot.String()+"/subdir/fnord", nil) + req.Header.Set("Accept", "application/json") res := mustDoWithoutRedirect(t, req) defer res.Body.Close() - assert.Equal(t, http.StatusOK, res.StatusCode) + assert.NotEqual(t, http.StatusNotAcceptable, res.StatusCode) }) } diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 122756423..1311095e8 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -57,6 +57,10 @@ var contentTypeToExtension = map[string]string{ dagCborResponseFormat: ".cbor", } +// errCodecConversionHint is the user-facing hint returned in 406 responses +// when codec conversion is not allowed (IPIP-0524). +const errCodecConversionHint = "codec conversion is not supported, fetch raw block with ?format=raw and convert client-side" + func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { ctx, span := spanTrace(ctx, "Handler.ServeCodec", trace.WithAttributes(attribute.String("path", rq.immutablePath.String()), attribute.String("requestedContentType", rq.responseFormat))) defer span.End() @@ -151,7 +155,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt // IPIP-0524: Check if codec conversion is allowed if !i.config.AllowCodecConversion && toCodec != cidCodec { // Conversion not allowed and codecs don't match - return 406 - err := fmt.Errorf("format %q requested but block has codec %q: codec conversion is not supported", rq.responseFormat, cidCodec.String()) + err := fmt.Errorf("format %q requested but block has codec %q: %s", rq.responseFormat, cidCodec.String(), errCodecConversionHint) i.webError(w, r, err, http.StatusNotAcceptable) return false } From 4c124a6f1642fd58e2c666fd6eb55bf65ca4388a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 5 Feb 2026 21:53:23 +0100 Subject: [PATCH 4/6] chore: update changelog with PR link for IPIP-0524 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9e610e2c..0622b5860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ The following emojis are used to highlight certain changes: - 🛠 `chunker`: `DefaultBlockSize` changed from `const` to `var` to allow runtime configuration via global profiles. [#1088](https://github.com/ipfs/boxo/pull/1088), [IPIP-499](https://github.com/ipfs/specs/pull/499) - `gateway`: 🛠 ✨ [IPIP-523](https://github.com/ipfs/specs/pull/523) `?format=` URL query parameter now takes precedence over `Accept` HTTP header, ensuring deterministic HTTP cache behavior and allowing browsers to use `?format=` even when they send `Accept` headers with specific content types. [#1074](https://github.com/ipfs/boxo/pull/1074) -- `gateway`: 🛠 ✨ [IPIP-0524](https://github.com/ipfs/specs/pull/524) codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default. Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable. Clients should fetch raw blocks (`?format=raw`) and convert in userland. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. +- `gateway`: 🛠 ✨ [IPIP-0524](https://github.com/ipfs/specs/pull/524) codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default. Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable with a hint to fetch raw blocks (`?format=raw`) and convert client-side. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. [#1077](https://github.com/ipfs/boxo/pull/1077) ### Removed From 446a00ba084c12939f66e2cd7c2a09860563fbab Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 6 Feb 2026 00:05:21 +0100 Subject: [PATCH 5/6] fix(gateway): show only native codec download link in HTML preview only show the matching format download link (dag-json or dag-cbor) in the HTML preview page when AllowCodecConversion is disabled, preventing users from clicking links that would return 406. when AllowCodecConversion is enabled, both links are shown as before. --- gateway/assets/assets.go | 11 +++++---- gateway/assets/dag.html | 6 ++--- gateway/handler_codec.go | 13 ++++++----- gateway/handler_codec_test.go | 44 +++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/gateway/assets/assets.go b/gateway/assets/assets.go index b541d21ba..d123cc6cf 100644 --- a/gateway/assets/assets.go +++ b/gateway/assets/assets.go @@ -96,11 +96,12 @@ type GlobalData struct { type DagTemplateData struct { GlobalData - Path string - CID string - CodecName string - CodecHex string - Node *ParsedNode + Path string + CID string + CodecName string + CodecHex string + Node *ParsedNode + AllowCodecConversion bool } type ErrorTemplateData struct { diff --git a/gateway/assets/dag.html b/gateway/assets/dag.html index 61d799a5b..ca681194b 100644 --- a/gateway/assets/dag.html +++ b/gateway/assets/dag.html @@ -23,9 +23,9 @@

You can download this block as:

{{ with .Node }} diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 1311095e8..2b21cdcd4 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -214,12 +214,13 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * cidCodec := mc.Code(resolvedPath.RootCid().Prefix().Codec) err = assets.DagTemplate.Execute(w, assets.DagTemplateData{ - GlobalData: i.getTemplateGlobalData(r, contentPath), - Path: contentPath.String(), - CID: resolvedPath.RootCid().String(), - CodecName: cidCodec.String(), - CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), - Node: parseNode(blockCid, blockData), + GlobalData: i.getTemplateGlobalData(r, contentPath), + Path: contentPath.String(), + CID: resolvedPath.RootCid().String(), + CodecName: cidCodec.String(), + CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + Node: parseNode(blockCid, blockData), + AllowCodecConversion: i.config.AllowCodecConversion, }) if err != nil { _, _ = fmt.Fprintf(w, "error during body generation: %v", err) diff --git a/gateway/handler_codec_test.go b/gateway/handler_codec_test.go index 127e0bc8c..90084f48b 100644 --- a/gateway/handler_codec_test.go +++ b/gateway/handler_codec_test.go @@ -79,4 +79,48 @@ func TestDagJsonCborPreview(t *testing.T) { require.Contains(t, string(body), escaped) require.NotContains(t, string(body), script) }) + + t.Run("download links without AllowCodecConversion", func(t *testing.T) { + t.Parallel() + + tsNoConv := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, + }) + + req := mustNewRequest(t, http.MethodGet, tsNoConv.URL+resolvedPath.String()+"/", nil) + req.Header.Add("Accept", "text/html") + + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusOK, res.StatusCode) + + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + + require.Contains(t, string(body), `href="?format=raw"`, "raw block link always present") + require.Contains(t, string(body), `href="?format=dag-cbor"`, "native codec link present") + require.NotContains(t, string(body), `href="?format=dag-json"`, "cross-codec link absent when conversion disabled") + }) + + t.Run("download links with AllowCodecConversion", func(t *testing.T) { + t.Parallel() + + tsConv := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: true, + }) + + req := mustNewRequest(t, http.MethodGet, tsConv.URL+resolvedPath.String()+"/", nil) + req.Header.Add("Accept", "text/html") + + res := mustDoWithoutRedirect(t, req) + require.Equal(t, http.StatusOK, res.StatusCode) + + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + + require.Contains(t, string(body), `href="?format=raw"`, "raw block link always present") + require.Contains(t, string(body), `href="?format=dag-cbor"`, "native codec link present") + require.Contains(t, string(body), `href="?format=dag-json"`, "cross-codec link present when conversion enabled") + }) } From a36d1f3acb59464189f39f64ad7f74eface44198 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 6 Feb 2026 00:18:44 +0100 Subject: [PATCH 6/6] ci: switch gateway-conformance to v0.10 --- .github/workflows/gateway-conformance.yml | 12 ++++++------ CHANGELOG.md | 3 ++- gateway/gateway.go | 2 +- gateway/gateway_test.go | 2 +- gateway/handler_codec.go | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 4fabae220..15e00383d 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10 with: output: fixtures merged: true @@ -47,7 +47,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/test@v0.10 with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +84,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10 with: output: fixtures merged: true @@ -114,7 +114,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/test@v0.10 with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +152,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10 with: output: fixtures merged: true @@ -182,7 +182,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@e17586f4cccdd0f93f19a68b66a25e07e03731f6 # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships + uses: ipfs/gateway-conformance/.github/actions/test@v0.10 with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0622b5860..b12ac141f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,8 @@ The following emojis are used to highlight certain changes: - 🛠 `chunker`: `DefaultBlockSize` changed from `const` to `var` to allow runtime configuration via global profiles. [#1088](https://github.com/ipfs/boxo/pull/1088), [IPIP-499](https://github.com/ipfs/specs/pull/499) - `gateway`: 🛠 ✨ [IPIP-523](https://github.com/ipfs/specs/pull/523) `?format=` URL query parameter now takes precedence over `Accept` HTTP header, ensuring deterministic HTTP cache behavior and allowing browsers to use `?format=` even when they send `Accept` headers with specific content types. [#1074](https://github.com/ipfs/boxo/pull/1074) -- `gateway`: 🛠 ✨ [IPIP-0524](https://github.com/ipfs/specs/pull/524) codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default. Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable with a hint to fetch raw blocks (`?format=raw`) and convert client-side. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. [#1077](https://github.com/ipfs/boxo/pull/1077) +- `gateway`: 🛠 ✨ [IPIP-524](https://github.com/ipfs/specs/pull/524) codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default. Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable with a hint to fetch raw blocks (`?format=raw`) and convert client-side. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. [#1077](https://github.com/ipfs/boxo/pull/1077) +- `gateway`: compliance with gateway-conformance [v0.10.0](https://github.com/ipfs/gateway-conformance/releases/tag/v0.10.0) (since v0.8: relaxed DAG-CBOR HTML preview cache headers, relaxed CAR 200/404 for missing paths, [IPIP-523](https://github.com/ipfs/specs/pull/523) format query precedence, [IPIP-524](https://github.com/ipfs/specs/pull/524) codec mismatch returns 406) ### Removed diff --git a/gateway/gateway.go b/gateway/gateway.go index ac3601f46..a22e3e4e9 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -62,7 +62,7 @@ type Config struct { // // When false (default), the gateway returns 406 Not Acceptable if the // requested format doesn't match the block's codec. This follows the - // behavior specified in IPIP-0524. + // behavior specified in IPIP-524. // // When true, the gateway attempts to convert between legacy IPLD formats. // This is provided for backwards compatibility but is not required by diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 29728e91d..476aca9e2 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -1255,7 +1255,7 @@ func TestAllowCodecConversion(t *testing.T) { {"dag-cbor block with dag-json", cborBackend, "/ipfs/" + dagCborRoot.String(), "dag-json"}, {"dag-pb directory with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/", "dag-json"}, {"dag-pb directory with dag-cbor", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/", "dag-cbor"}, - {"dag-pb file with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/fnord", "dag-json"}, + {"dag-pb file with dag-json", pbBackend, "/ipfs/bafyaacqkbaeaeeqcpn6rqaq", "dag-json"}, {"raw block with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/fnord", "dag-json"}, } { t.Run("AllowCodecConversion=false returns 406 for "+tc.name, func(t *testing.T) { diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 2b21cdcd4..414c89900 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -58,7 +58,7 @@ var contentTypeToExtension = map[string]string{ } // errCodecConversionHint is the user-facing hint returned in 406 responses -// when codec conversion is not allowed (IPIP-0524). +// when codec conversion is not allowed (IPIP-524). const errCodecConversionHint = "codec conversion is not supported, fetch raw block with ?format=raw and convert client-side" func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData) bool { @@ -152,7 +152,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return false } - // IPIP-0524: Check if codec conversion is allowed + // IPIP-524: Check if codec conversion is allowed if !i.config.AllowCodecConversion && toCodec != cidCodec { // Conversion not allowed and codecs don't match - return 406 err := fmt.Errorf("format %q requested but block has codec %q: %s", rq.responseFormat, cidCodec.String(), errCodecConversionHint)