From 70904d7165be4928200f1b9768f7161f6f7cf648 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 17 Apr 2024 10:32:31 +0200 Subject: [PATCH 1/2] gateway: add support for car-* query parameters --- CHANGELOG.md | 1 + gateway/gateway_test.go | 2 +- gateway/handler.go | 16 +++++++--------- gateway/handler_car.go | 24 +++++++++++++++++++++--- gateway/handler_car_test.go | 20 ++++++++++++-------- 5 files changed, 42 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca083b47f..12eb3eec9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The following emojis are used to highlight certain changes: * `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`. * `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`. * `gateway` now sets the [`Content-Location`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Location) header for requests with non-default content format, as a result of content negotiation. This allows generic and misconfigured HTTP caches to store Deserialized, CAR and Block responses separately, under distinct cache keys. +* `gateway` now supports `car-dups`, `car-order` and `car-version` as query parameters in addition to the `Accept` header parameters. The parameters in the `Accept` header have always priority. ### Changed diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 391447d02..d59f0d66f 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -470,7 +470,7 @@ func TestHeaders(t *testing.T) { dnslinkGatewayHost := "dnslink-gateway.com" runTest("Regular gateway with default format", contentPath, "", "", "") - runTest("Regular gateway with Accept: application/vnd.ipld.car has no Content-Location", contentPath, "application/vnd.ipld.car;version=1;order=dfs;dups=n", "", "") + runTest("Regular gateway with Accept: application/vnd.ipld.car;version=1;order=dfs;dups=n sets correct Content-Location", contentPath, "application/vnd.ipld.car;version=1;order=dfs;dups=n", "", contentPath+"?car-dups=n&car-order=dfs&car-version=1&format=car") runTest("Regular gateway with ?dag-scope=entity&format=car", contentPath+"?dag-scope=entity&format=car", "", "", "") runTest("Regular gateway preserves query parameters", contentPath+"?a=b&c=d", dagCborResponseFormat, "", contentPath+"?a=b&c=d&format=dag-cbor") runTest("Subdomain gateway with default format", "/empty-dir/", "", subdomainGatewayHost, "") diff --git a/gateway/handler.go b/gateway/handler.go index d8aa82e5e..522af1eb5 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -658,14 +658,7 @@ func addContentLocation(r *http.Request, w http.ResponseWriter, rq *requestData) return } - // Response format parameters, such as 'dups' and 'order' for CAR requests - // cannot be translated into the URL. Therefore, we cannot add a 'Content-Location' - // header. - if len(rq.responseParams) != 0 { - return - } - - param := responseFormatToFormatParam[rq.responseFormat] + format := responseFormatToFormatParam[rq.responseFormat] path := r.URL.Path if p, ok := r.Context().Value(OriginalPathKey).(string); ok { path = p @@ -676,7 +669,12 @@ func addContentLocation(r *http.Request, w http.ResponseWriter, rq *requestData) for k, v := range r.URL.Query() { query[k] = v } - query.Set("format", param) + query.Set("format", format) + + // Set response params as query elements. + for k, v := range rq.responseParams { + query.Set(format+"-"+k, v) + } w.Header().Set("Content-Location", path+"?"+query.Encode()) } diff --git a/gateway/handler_car.go b/gateway/handler_car.go index e42c0fde2..9c63d2d16 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -23,6 +23,9 @@ import ( const ( carRangeBytesKey = "entity-bytes" carTerminalElementTypeKey = "dag-scope" + carVersionKey = "car-version" + carDuplicatesKey = "car-dups" + carOrderKey = "car-order" ) // serveCAR returns a CAR stream for specific DAG+selector @@ -144,8 +147,23 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa // application/vnd.ipld.car content type parameters from Accept header + // Get CAR version, duplicates and order from the query parameters and override + // with parameters from Accept header if they exist, since they have priority. + versionStr := queryParams.Get(carVersionKey) + duplicatesStr := queryParams.Get(carDuplicatesKey) + orderStr := queryParams.Get(carOrderKey) + if v, ok := contentTypeParams["version"]; ok { + versionStr = v + } + if v, ok := contentTypeParams["order"]; ok { + orderStr = v + } + if v, ok := contentTypeParams["dups"]; ok { + duplicatesStr = v + } + // version of CAR format - switch contentTypeParams["version"] { + switch versionStr { case "": // noop, client does not care about version case "1": // noop, we support this default: @@ -153,7 +171,7 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa } // optional order from IPIP-412 - if order := DagOrder(contentTypeParams["order"]); order != DagOrderUnspecified { + if order := DagOrder(orderStr); order != DagOrderUnspecified { switch order { case DagOrderUnknown, DagOrderDFS: params.Order = order @@ -168,7 +186,7 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa } // optional dups from IPIP-412 - dups, err := NewDuplicateBlocksPolicy(contentTypeParams["dups"]) + dups, err := NewDuplicateBlocksPolicy(duplicatesStr) if err != nil { return CarParams{}, err } diff --git a/gateway/handler_car_test.go b/gateway/handler_car_test.go index da2d16255..a9c3667d2 100644 --- a/gateway/handler_car_test.go +++ b/gateway/handler_car_test.go @@ -2,6 +2,7 @@ package gateway import ( "net/http" + "net/url" "testing" "github.com/ipfs/boxo/path" @@ -81,19 +82,22 @@ func TestCarParams(t *testing.T) { // from the value read from Accept header tests := []struct { acceptHeader string + params url.Values expectedOrder DagOrder expectedDuplicates DuplicateBlocksPolicy }{ - {"application/vnd.ipld.car; order=dfs; dups=y", DagOrderDFS, DuplicateBlocksIncluded}, - {"application/vnd.ipld.car; order=unk; dups=n", DagOrderUnknown, DuplicateBlocksExcluded}, - {"application/vnd.ipld.car; order=unk", DagOrderUnknown, DuplicateBlocksExcluded}, - {"application/vnd.ipld.car; dups=y", DagOrderDFS, DuplicateBlocksIncluded}, - {"application/vnd.ipld.car; dups=n", DagOrderDFS, DuplicateBlocksExcluded}, - {"application/vnd.ipld.car", DagOrderDFS, DuplicateBlocksExcluded}, - {"application/vnd.ipld.car;version=1;order=dfs;dups=y", DagOrderDFS, DuplicateBlocksIncluded}, + {"application/vnd.ipld.car; order=dfs; dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded}, + {"application/vnd.ipld.car; order=unk; dups=n", nil, DagOrderUnknown, DuplicateBlocksExcluded}, + {"application/vnd.ipld.car; order=unk", nil, DagOrderUnknown, DuplicateBlocksExcluded}, + {"application/vnd.ipld.car; dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded}, + {"application/vnd.ipld.car; dups=n", nil, DagOrderDFS, DuplicateBlocksExcluded}, + {"application/vnd.ipld.car", nil, DagOrderDFS, DuplicateBlocksExcluded}, + {"application/vnd.ipld.car;version=1;order=dfs;dups=y", nil, DagOrderDFS, DuplicateBlocksIncluded}, + {"application/vnd.ipld.car;version=1;order=dfs;dups=y", url.Values{"car-order": []string{"unk"}}, DagOrderDFS, DuplicateBlocksIncluded}, + {"application/vnd.ipld.car;version=1;dups=y", url.Values{"car-order": []string{"unk"}}, DagOrderUnknown, DuplicateBlocksIncluded}, } for _, test := range tests { - r := mustNewRequest(t, http.MethodGet, "http://example.com/", nil) + r := mustNewRequest(t, http.MethodGet, "http://example.com/?"+test.params.Encode(), nil) r.Header.Set("Accept", test.acceptHeader) mediaType, formatParams, err := customResponseFormat(r) From 65cd4013b4509bc209a276a0fb4a4604e870307b Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 18 Apr 2024 00:35:49 +0200 Subject: [PATCH 2/2] docs: CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12eb3eec9..1efccae44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ The following emojis are used to highlight certain changes: * `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`. * `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`. * `gateway` now sets the [`Content-Location`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Location) header for requests with non-default content format, as a result of content negotiation. This allows generic and misconfigured HTTP caches to store Deserialized, CAR and Block responses separately, under distinct cache keys. -* `gateway` now supports `car-dups`, `car-order` and `car-version` as query parameters in addition to the `Accept` header parameters. The parameters in the `Accept` header have always priority. +* `gateway` now supports `car-dups`, `car-order` and `car-version` as query parameters in addition to the `application/vnd.ipld.car` parameters sent via `Accept` header. The parameters in the `Accept` header have always priority, but including them in URL simplifies HTTP caching and allows use in `Content-Location` header on CAR responses to maximize interoperability with wide array of HTTP caches. ### Changed