diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 84f5415a9c9..8efc41424db 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -41,7 +41,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.9 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10 with: output: fixtures @@ -93,7 +93,7 @@ jobs: # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests - uses: ipfs/gateway-conformance/.github/actions/test@v0.9 + uses: ipfs/gateway-conformance/.github/actions/test@v0.10 with: gateway-url: http://127.0.0.1:8080 subdomain-url: http://localhost:8080 @@ -127,7 +127,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.9 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.10 with: output: fixtures @@ -199,7 +199,7 @@ jobs: # 9. Run the gateway-conformance tests over libp2p - name: Run gateway-conformance tests over libp2p - uses: ipfs/gateway-conformance/.github/actions/test@v0.9 + uses: ipfs/gateway-conformance/.github/actions/test@v0.10 with: gateway-url: http://127.0.0.1:8092 args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' diff --git a/config/gateway.go b/config/gateway.go index 0ce7638f4f5..bef74633ac1 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -10,6 +10,7 @@ const ( DefaultDisableHTMLErrors = false DefaultExposeRoutingAPI = true DefaultDiagnosticServiceURL = "https://check.ipfs.network" + DefaultAllowCodecConversion = false // Gateway limit defaults from boxo DefaultRetrievalTimeout = gateway.DefaultRetrievalTimeout @@ -73,6 +74,12 @@ type Gateway struct { // be overridden per FQDN in PublicGateways. DeserializedResponses Flag + // AllowCodecConversion enables automatic conversion between codecs when + // the requested format differs from the block's native codec (e.g., + // converting dag-pb or dag-cbor to dag-json). When disabled, the gateway + // returns 406 Not Acceptable for codec mismatches per IPIP-524. + AllowCodecConversion Flag + // DisableHTMLErrors disables pretty HTML pages when an error occurs. Instead, a `text/plain` // page will be sent with the raw error message. DisableHTMLErrors Flag diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index 6bac662aea8..f8479cb446f 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -269,6 +269,7 @@ func getGatewayConfig(n *core.IpfsNode) (gateway.Config, map[string][]string, er // Initialize gateway configuration, with empty PublicGateways, handled after. gwCfg := gateway.Config{ DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses), + AllowCodecConversion: cfg.Gateway.AllowCodecConversion.WithDefault(config.DefaultAllowCodecConversion), DisableHTMLErrors: cfg.Gateway.DisableHTMLErrors.WithDefault(config.DefaultDisableHTMLErrors), NoDNSLink: cfg.Gateway.NoDNSLink, PublicGateways: map[string]*gateway.PublicGateway{}, diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 459b4928f9f..f99ca0e9586 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -15,6 +15,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Routing V1 HTTP API now exposed by default](#routing-v1-http-api-now-exposed-by-default) - [Track total size when adding pins](#track-total-size-when-adding-pins) - [IPIP-523: `?format=` takes precedence over `Accept` header](#ipip-523-format-takes-precedence-over-accept-header) + - [IPIP-524: Gateway codec conversion disabled by default](#ipip-524-gateway-codec-conversion-disabled-by-default) - [Improved IPNS over PubSub validation](#improved-ipns-over-pubsub-validation) - [New `ipfs diag datastore` commands](#new-ipfs-diag-datastore-commands) - [🚇 Improved `ipfs p2p` tunnels with foreground mode](#-improved-ipfs-p2p-tunnels-with-foreground-mode) @@ -99,6 +100,18 @@ This ensures deterministic HTTP caching behavior and protects against CDNs that The only breaking change is for edge cases where a client sends both a specific `Accept` header and a different `?format=` value for an explicitly supported format (`tar`, `raw`, `car`, `dag-json`, `dag-cbor`, etc.). Previously `Accept` would win. Now `?format=` always wins. +#### IPIP-524: Gateway codec conversion disabled by default + +Codec conversion is now disabled by default per [IPIP-524](https://github.com/ipfs/specs/pull/524). +Requests for a format that differs from the block's codec will return `406 Not Acceptable`. + +**Migration**: Clients should fetch raw blocks (`?format=raw` or `Accept: application/vnd.ipld.raw`) +and convert client-side using libraries like [@helia/verified-fetch](https://www.npmjs.com/package/@helia/verified-fetch). + +This change removes gateways from a gatekeeping role: new codecs can now be adopted by clients +immediately without waiting for gateway operator updates. Set [`Gateway.AllowCodecConversion`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayallowcodecconversion) +to `true` to restore previous behavior. + #### Improved IPNS over PubSub validation [IPNS over PubSub](https://specs.ipfs.tech/ipns/ipns-pubsub-router/) implementation in Kubo is now more reliable. Duplicate messages are rejected even in large networks where messages may cycle back after the in-memory cache expires. @@ -262,7 +275,7 @@ The Inspect button now resolves `/ipfs/` and `/ipns/` paths to their final CID b - update `boxo` to [v0.36.0](https://github.com/ipfs/boxo/releases/tag/v0.36.0) - update `go-libp2p-kad-dht` to [v0.37.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.37.1) (includes [v0.37.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.37.0)) - update `ipfs-webui` to [v4.11.0](https://github.com/ipfs/ipfs-webui/releases/tag/v4.11.0) -- update `gateway-conformance` tests to [v0.9](https://github.com/ipfs/gateway-conformance/releases/tag/v0.9.0) +- update `gateway-conformance` tests to [v0.10](https://github.com/ipfs/gateway-conformance/releases/tag/v0.10.0) (incl. [v0.9](https://github.com/ipfs/gateway-conformance/releases/tag/v0.9.0)) ### 📝 Changelog diff --git a/docs/config.md b/docs/config.md index 843686b9569..4d3a44bb871 100644 --- a/docs/config.md +++ b/docs/config.md @@ -64,6 +64,7 @@ config file at runtime. - [`Gateway.NoFetch`](#gatewaynofetch) - [`Gateway.NoDNSLink`](#gatewaynodnslink) - [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses) + - [`Gateway.AllowCodecConversion`](#gatewayallowcodecconversion) - [`Gateway.DisableHTMLErrors`](#gatewaydisablehtmlerrors) - [`Gateway.ExposeRoutingAPI`](#gatewayexposeroutingapi) - [`Gateway.RetrievalTimeout`](#gatewayretrievaltimeout) @@ -1141,6 +1142,32 @@ Default: `true` Type: `flag` +### `Gateway.AllowCodecConversion` + +An optional flag to enable automatic conversion between codecs when the +requested format differs from the block's native codec (e.g., converting +dag-pb or dag-cbor to dag-json). + +When disabled (the default), the gateway returns `406 Not Acceptable` for +codec mismatches, following behavior specified in +[IPIP-524](https://github.com/ipfs/specs/pull/524). + +Most users should keep this disabled unless legacy +[IPLD Logical Format](https://web.archive.org/web/20260204204727/https://ipld.io/specs/codecs/dag-pb/spec/#logical-format) +support is needed as a stop-gap while switching clients to `?format=raw` +and converting client-side. + +Instead of relying on gateway-side conversion, fetch the raw block using +`?format=raw` (`application/vnd.ipld.raw`) and convert client-side. This: + +- Allows clients to use any codec without waiting for gateway support +- Enables ecosystem innovation without gateway operator coordination +- Works with libraries like [@helia/verified-fetch](https://www.npmjs.com/package/@helia/verified-fetch) in JavaScript + +Default: `false` + +Type: `flag` + ### `Gateway.DisableHTMLErrors` An optional flag to disable the pretty HTML error pages of the gateway. Instead, diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 322e0cb0a9d..4d47845f9a8 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.25 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412 + github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.47.0 github.com/multiformats/go-multiaddr v0.16.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 1703c15965b..f2accefea6b 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -267,8 +267,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412 h1:nfRIkMIhetCWD8jw5ya+FY+jn9ii2c+U5gdkmSS4L1Q= -github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75 h1:1UoSAzXwwgOrCZm5cu6v6bL4OGYIzcaOew9Rl6ZycqQ= +github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/go.mod b/go.mod index ebe83b9bb63..d56bc3649fa 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/hashicorp/go-version v1.8.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412 + github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 5055dcb40be..62fdd089362 100644 --- a/go.sum +++ b/go.sum @@ -337,8 +337,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412 h1:nfRIkMIhetCWD8jw5ya+FY+jn9ii2c+U5gdkmSS4L1Q= -github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75 h1:1UoSAzXwwgOrCZm5cu6v6bL4OGYIzcaOew9Rl6ZycqQ= +github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 43ca71ed35f..ea49b63a560 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -135,7 +135,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412 // indirect + github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect github.com/ipfs/go-cid v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 120b95acba8..2b4fe150880 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -296,8 +296,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412 h1:nfRIkMIhetCWD8jw5ya+FY+jn9ii2c+U5gdkmSS4L1Q= -github.com/ipfs/boxo v0.36.1-0.20260204203152-f188f79fd412/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= +github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75 h1:1UoSAzXwwgOrCZm5cu6v6bL4OGYIzcaOew9Rl6ZycqQ= +github.com/ipfs/boxo v0.36.1-0.20260205235512-2a942e3e1a75/go.mod h1:92hnRXfP5ScKEIqlq9Ns7LR1dFXEVADKWVGH0fjk83k= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.2.3 h1:mpCuDaNXJ4wrBJLrtEaGFGXkferrw5eqVvzaHhtFKQk=