feat(gateway): IPIP-0524 + AllowCodecConversion config option

.github/workflows/gateway-conformance.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.9
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10
         with:
           output: fixtures
           merged: true
@@ -47,7 +47,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.9
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.10
         with:
           gateway-url: http://127.0.0.1:8040
           subdomain-url: http://example.net:8040
@@ -84,7 +84,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.9
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10
         with:
           output: fixtures
           merged: true
@@ -114,7 +114,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.9
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.10
         with:
           gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
           subdomain-url: http://example.net:8040
@@ -152,7 +152,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.9
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10
         with:
           output: fixtures
           merged: true
@@ -182,7 +182,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.9
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.10
         with:
           gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
           subdomain-url: http://example.net:8040

CHANGELOG.md

@@ -23,7 +23,9 @@ The following emojis are used to highlight certain changes:
 ### Changed
 
 - 🛠 `chunker`: `DefaultBlockSize` changed from `const` to `var` to allow runtime configuration via global profiles. [#1088](https://github.com/ipfs/boxo/pull/1088), [IPIP-499](https://github.com/ipfs/specs/pull/499)
-- `gateway`: ✨ [IPIP-523](https://github.com/ipfs/specs/pull/523) `?format=` URL query parameter now takes precedence over `Accept` HTTP header, ensuring deterministic HTTP cache behavior and allowing browsers to use `?format=` even when they send `Accept` headers with specific content types. [#1074](https://github.com/ipfs/boxo/pull/1074)
+- `gateway`: 🛠 ✨ [IPIP-523](https://github.com/ipfs/specs/pull/523) `?format=` URL query parameter now takes precedence over `Accept` HTTP header, ensuring deterministic HTTP cache behavior and allowing browsers to use `?format=` even when they send `Accept` headers with specific content types. [#1074](https://github.com/ipfs/boxo/pull/1074)
+- `gateway`: 🛠 ✨ [IPIP-524](https://github.com/ipfs/specs/pull/524) codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default. Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable with a hint to fetch raw blocks (`?format=raw`) and convert client-side. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. [#1077](https://github.com/ipfs/boxo/pull/1077)
+- `gateway`: compliance with gateway-conformance [v0.10.0](https://github.com/ipfs/gateway-conformance/releases/tag/v0.10.0) (since v0.8: relaxed DAG-CBOR HTML preview cache headers, relaxed CAR 200/404 for missing paths, [IPIP-523](https://github.com/ipfs/specs/pull/523) format query precedence, [IPIP-524](https://github.com/ipfs/specs/pull/524) codec mismatch returns 406)
 
 ### Removed
 

examples/gateway/car-file/main_test.go

@@ -1,15 +1,14 @@
 package main
 
 import (
+	"bytes"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/ipfs/boxo/examples/gateway/common"
 	"github.com/ipfs/boxo/gateway"
-	"github.com/ipld/go-ipld-prime/codec/dagjson"
-	"github.com/ipld/go-ipld-prime/node/basicnode"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -62,48 +61,35 @@ func TestFile(t *testing.T) {
 	assert.EqualValues(t, string(body), "hello world\n")
 }
 
-func TestDirectoryAsDAG(t *testing.T) {
+func TestDirectoryAsRawBlock(t *testing.T) {
 	ts, f, err := newTestServer()
 	assert.NoError(t, err)
 	defer f.Close()
 
-	res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=dag-json")
+	res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=raw")
 	assert.NoError(t, err)
 	defer res.Body.Close()
 
-	contentType := res.Header.Get("Content-Type")
-	assert.EqualValues(t, contentType, "application/vnd.ipld.dag-json")
-
-	// Parses the DAG-JSON response.
-	dag := basicnode.Prototype.Any.NewBuilder()
-	err = dagjson.Decode(dag, res.Body)
-	assert.NoError(t, err)
-
-	// Checks for the links inside the logical model.
-	links, err := dag.Build().LookupByString("Links")
-	assert.NoError(t, err)
-
-	// Checks if there are 2 links.
-	assert.EqualValues(t, links.Length(), 2)
-
-	// Check if the first item is correct.
-	n, err := links.LookupByIndex(0)
-	assert.NoError(t, err)
-	assert.NotNil(t, n)
+	assert.Equal(t, http.StatusOK, res.StatusCode)
 
-	nameNode, err := n.LookupByString("Name")
-	assert.NoError(t, err)
-	assert.NotNil(t, nameNode)
-
-	name, err := nameNode.AsString()
-	assert.NoError(t, err)
-	assert.EqualValues(t, name, "eye.png")
+	contentType := res.Header.Get("Content-Type")
+	assert.Equal(t, "application/vnd.ipld.raw", contentType)
 
-	hashNode, err := n.LookupByString("Hash")
+	body, err := io.ReadAll(res.Body)
 	assert.NoError(t, err)
-	assert.NotNil(t, hashNode)
 
-	hash, err := hashNode.AsLink()
-	assert.NoError(t, err)
-	assert.EqualValues(t, hash.String(), "bafybeigmlfksb374fdkxih4urny2yiyazyra2375y2e4a72b3jcrnthnau")
+	// Raw bytes of the dag-pb directory block
+	expected := []byte{
+		0x12, 0x33, 0x0a, 0x24, 0x01, 0x70, 0x12, 0x20, 0xcc, 0x59, 0x55, 0x20,
+		0xef, 0xfc, 0x28, 0xd5, 0x74, 0x1f, 0x94, 0x8b, 0x71, 0xac, 0x23, 0x00,
+		0xce, 0x22, 0x0d, 0x6f, 0xfd, 0xc6, 0x89, 0xc0, 0x7f, 0x41, 0xda, 0x45,
+		0x16, 0xcc, 0xed, 0x05, 0x12, 0x07, 0x65, 0x79, 0x65, 0x2e, 0x70, 0x6e,
+		0x67, 0x18, 0xd0, 0xc8, 0x10, 0x12, 0x33, 0x0a, 0x24, 0x01, 0x55, 0x12,
+		0x20, 0xa9, 0x48, 0x90, 0x4f, 0x2f, 0x0f, 0x47, 0x9b, 0x8f, 0x81, 0x97,
+		0x69, 0x4b, 0x30, 0x18, 0x4b, 0x0d, 0x2e, 0xd1, 0xc1, 0xcd, 0x2a, 0x1e,
+		0xc0, 0xfb, 0x85, 0xd2, 0x99, 0xa1, 0x92, 0xa4, 0x47, 0x12, 0x09, 0x68,
+		0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x74, 0x78, 0x74, 0x18, 0x0c, 0x0a, 0x02,
+		0x08, 0x01,
+	}
+	assert.True(t, bytes.Equal(body, expected), "raw block bytes should match")
 }

examples/go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/ipfs/go-cid v0.6.0
 	github.com/ipfs/go-datastore v0.9.0
 	github.com/ipld/go-car/v2 v2.16.0
-	github.com/ipld/go-ipld-prime v0.21.0
 	github.com/libp2p/go-libp2p v0.47.0
 	github.com/multiformats/go-multiaddr v0.16.1
 	github.com/multiformats/go-multicodec v0.10.0
@@ -63,6 +62,7 @@ require (
 	github.com/ipfs/go-peertaskqueue v0.8.3 // indirect
 	github.com/ipfs/go-unixfsnode v1.10.2 // indirect
 	github.com/ipld/go-codec-dagpb v1.7.0 // indirect
+	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect

gateway/assets/assets.go

@@ -96,11 +96,12 @@ type GlobalData struct {
 
 type DagTemplateData struct {
 	GlobalData
-	Path      string
-	CID       string
-	CodecName string
-	CodecHex  string
-	Node      *ParsedNode
+	Path                 string
+	CID                  string
+	CodecName            string
+	CodecHex             string
+	Node                 *ParsedNode
+	AllowCodecConversion bool
 }
 
 type ErrorTemplateData struct {

gateway/assets/dag.html

@@ -23,9 +23,9 @@
       <p>You can download this block as:</p>
 
       <ul>
-        <li><a href="?format=raw" rel="nofollow">Raw Block</a> (no conversion)</li>
-        <li><a href="?format=dag-json" rel="nofollow">Valid DAG-JSON</a> (specs at <a href="https://ipld.io/specs/codecs/dag-json/spec/" target="_blank" rel="noopener noreferrer">IPLD</a> and <a href="https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-json" target="_blank" rel="noopener noreferrer">IANA</a>)</li>
-        <li><a href="?format=dag-cbor" rel="nofollow">Valid DAG-CBOR</a> (specs at <a href="https://ipld.io/specs/codecs/dag-cbor/spec/" target="_blank" rel="noopener noreferrer">IPLD</a> and <a href="https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-cbor" target="_blank" rel="noopener noreferrer">IANA</a>)</li>
+        <li><a href="?format=raw" rel="nofollow">Raw Block</a></li>
+        {{ if or (eq .CodecName "dag-json") .AllowCodecConversion }}<li><a href="?format=dag-json" rel="nofollow">DAG-JSON</a> (specs at <a href="https://ipld.io/specs/codecs/dag-json/spec/" target="_blank" rel="noopener noreferrer">IPLD</a> and <a href="https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-json" target="_blank" rel="noopener noreferrer">IANA</a>)</li>{{ end }}
+        {{ if or (eq .CodecName "dag-cbor") .AllowCodecConversion }}<li><a href="?format=dag-cbor" rel="nofollow">DAG-CBOR</a> (specs at <a href="https://ipld.io/specs/codecs/dag-cbor/spec/" target="_blank" rel="noopener noreferrer">IPLD</a> and <a href="https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-cbor" target="_blank" rel="noopener noreferrer">IANA</a>)</li>{{ end }}
       </ul>
     </section>
     {{ with .Node }}

gateway/gateway.go

@@ -56,6 +56,19 @@ type Config struct {
 	// [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/
 	DeserializedResponses bool
 
+	// AllowCodecConversion enables automatic conversion between codecs when
+	// the requested format differs from the block's native codec. For example,
+	// converting dag-pb (UnixFS) to dag-json.
+	//
+	// When false (default), the gateway returns 406 Not Acceptable if the
+	// requested format doesn't match the block's codec. This follows the
+	// behavior specified in IPIP-524.
+	//
+	// When true, the gateway attempts to convert between legacy IPLD formats.
+	// This is provided for backwards compatibility but is not required by
+	// the gateway specification.
+	AllowCodecConversion bool
+
 	// NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in
 	// response to requests with values in `Host` HTTP header. This flag can be
 	// overridden per FQDN in PublicGateways. To be used with WithHostname.

gateway/gateway_test.go

@@ -522,6 +522,7 @@ func TestHeaders(t *testing.T) {
 				},
 			},
 			DeserializedResponses: true,
+			AllowCodecConversion:  true, // Test tests various format conversions
 		})
 
 		runTest := func(name, path, accept, host, expectedContentLocationHdr string) {
@@ -1097,7 +1098,8 @@ func TestDeserializedResponses(t *testing.T) {
 		backend, root := newMockBackend(t, "fixtures.car")
 
 		ts := newTestServerWithConfig(t, backend, Config{
-			NoDNSLink: false,
+			NoDNSLink:            false,
+			AllowCodecConversion: true, // Test expects codec conversions to work
 			PublicGateways: map[string]*PublicGateway{
 				"trustless.com": {
 					Paths: []string{"/ipfs", "/ipns"},
@@ -1176,7 +1178,8 @@ func TestDeserializedResponses(t *testing.T) {
 		backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0)
 
 		ts := newTestServerWithConfig(t, backend, Config{
-			NoDNSLink: false,
+			NoDNSLink:            false,
+			AllowCodecConversion: true, // Test expects codec conversions to work
 			PublicGateways: map[string]*PublicGateway{
 				"trustless.com": {
 					Paths: []string{"/ipfs", "/ipns"},
@@ -1210,6 +1213,110 @@ func TestDeserializedResponses(t *testing.T) {
 	})
 }
 
+func TestAllowCodecConversion(t *testing.T) {
+	t.Parallel()
+
+	cborBackend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car")
+	pbBackend, dagPbRoot := newMockBackend(t, "fixtures.car")
+
+	// Positive cases: matching codec or conversion enabled should return 200
+	t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) {
+		t.Parallel()
+		ts := newTestServerWithConfig(t, cborBackend, Config{
+			DeserializedResponses: true,
+			AllowCodecConversion:  false,
+		})
+		req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil)
+		res := mustDoWithoutRedirect(t, req)
+		defer res.Body.Close()
+		assert.Equal(t, http.StatusOK, res.StatusCode)
+	})
+
+	t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) {
+		t.Parallel()
+		ts := newTestServerWithConfig(t, cborBackend, Config{
+			DeserializedResponses: true,
+			AllowCodecConversion:  true,
+		})
+		req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil)
+		res := mustDoWithoutRedirect(t, req)
+		defer res.Body.Close()
+		assert.Equal(t, http.StatusOK, res.StatusCode)
+	})
+
+	// Negative cases: requesting dag-json or dag-cbor for a block with a
+	// different codec should return 406 with an actionable error hint.
+	for _, tc := range []struct {
+		name    string
+		backend IPFSBackend
+		path    string
+		format  string
+	}{
+		{"dag-cbor block with dag-json", cborBackend, "/ipfs/" + dagCborRoot.String(), "dag-json"},
+		{"dag-pb directory with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/", "dag-json"},
+		{"dag-pb directory with dag-cbor", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/", "dag-cbor"},
+		{"dag-pb file with dag-json", pbBackend, "/ipfs/bafyaacqkbaeaeeqcpn6rqaq", "dag-json"},
+		{"raw block with dag-json", pbBackend, "/ipfs/" + dagPbRoot.String() + "/subdir/fnord", "dag-json"},
+	} {
+		t.Run("AllowCodecConversion=false returns 406 for "+tc.name, func(t *testing.T) {
+			t.Parallel()
+			ts := newTestServerWithConfig(t, tc.backend, Config{
+				DeserializedResponses: true,
+				AllowCodecConversion:  false,
+			})
+			req := mustNewRequest(t, http.MethodGet, ts.URL+tc.path+"?format="+tc.format, nil)
+			res := mustDoWithoutRedirect(t, req)
+			defer res.Body.Close()
+
+			body, err := io.ReadAll(res.Body)
+			require.NoError(t, err)
+			assert.Equal(t, http.StatusNotAcceptable, res.StatusCode)
+			assert.Contains(t, string(body), errCodecConversionHint)
+		})
+	}
+
+	// Ensure ?format=json on dag-pb and raw content serves the default
+	// response (not 406). JSON files may be stored as dag-pb or raw (UnixFS)
+	// and ?format=json goes through serveDefaults which does not do codec
+	// conversion - it serves directory listings or file bytes as-is.
+	for _, tc := range []struct {
+		name string
+		path string
+	}{
+		{"dag-pb directory", "/ipfs/" + dagPbRoot.String() + "/subdir/"},
+		{"dag-pb file", "/ipfs/" + dagPbRoot.String() + "/subdir/fnord"},
+	} {
+		t.Run("AllowCodecConversion=false serves default response for "+tc.name+" with json format", func(t *testing.T) {
+			t.Parallel()
+			ts := newTestServerWithConfig(t, pbBackend, Config{
+				DeserializedResponses: true,
+				AllowCodecConversion:  false,
+			})
+			req := mustNewRequest(t, http.MethodGet, ts.URL+tc.path+"?format=json", nil)
+			res := mustDoWithoutRedirect(t, req)
+			defer res.Body.Close()
+			assert.NotEqual(t, http.StatusNotAcceptable, res.StatusCode)
+		})
+	}
+
+	// Ensure dag-pb file with Accept: application/json is not rejected.
+	// This guards behavior where JSON files are stored as dag-pb or raw
+	// (UnixFS) and requested by clients with Accept: application/json.
+	// Regular HTTP use must not be broken by overreaching HTTP 406 from IPIP-524.
+	t.Run("AllowCodecConversion=false allows dag-pb file with Accept application/json", func(t *testing.T) {
+		t.Parallel()
+		ts := newTestServerWithConfig(t, pbBackend, Config{
+			DeserializedResponses: true,
+			AllowCodecConversion:  false,
+		})
+		req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagPbRoot.String()+"/subdir/fnord", nil)
+		req.Header.Set("Accept", "application/json")
+		res := mustDoWithoutRedirect(t, req)
+		defer res.Body.Close()
+		assert.NotEqual(t, http.StatusNotAcceptable, res.StatusCode)
+	})
+}
+
 type errorMockBackend struct {
 	err error
 }