diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index fdc91ae4c37..75dba80c769 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -2,24 +2,26 @@ package corehttp import ( "context" + "errors" "fmt" "io" "net" "net/http" + "github.com/ipfs/boxo/blockservice" iface "github.com/ipfs/boxo/coreiface" - options "github.com/ipfs/boxo/coreiface/options" - nsopts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway" "github.com/ipfs/boxo/namesys" - "github.com/ipfs/go-block-format" + offlineroute "github.com/ipfs/boxo/routing/offline" cid "github.com/ipfs/go-cid" version "github.com/ipfs/kubo" config "github.com/ipfs/kubo/config" core "github.com/ipfs/kubo/core" - coreapi "github.com/ipfs/kubo/core/coreapi" + "github.com/ipfs/kubo/core/node" + "github.com/libp2p/go-libp2p/core/routing" id "github.com/libp2p/go-libp2p/p2p/protocol/identify" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) @@ -42,7 +44,7 @@ func GatewayOption(paths ...string) ServeOption { Headers: headers, } - gwAPI, err := newGatewayAPI(n) + gwAPI, err := newGatewayBackend(n) if err != nil { return nil, err } @@ -70,7 +72,7 @@ func HostnameOption() ServeOption { return nil, err } - gwAPI, err := newGatewayAPI(n) + gwAPI, err := newGatewayBackend(n) if err != nil { return nil, err } @@ -93,83 +95,120 @@ func VersionOption() ServeOption { } } -type gatewayAPI struct { - ns namesys.NameSystem - api iface.CoreAPI - offlineAPI iface.CoreAPI -} - -func newGatewayAPI(n *core.IpfsNode) (*gatewayAPI, error) { +func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { cfg, err := n.Repo.Config() if err != nil { return nil, err } - api, err := coreapi.NewCoreAPI(n, options.Api.FetchBlocks(!cfg.Gateway.NoFetch)) - if err != nil { - return nil, err + bserv := n.Blocks + var vsRouting routing.ValueStore = n.Routing + nsys := n.Namesys + if cfg.Gateway.NoFetch { + bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore())) + + cs := cfg.Ipns.ResolveCacheSize + if cs == 0 { + cs = node.DefaultIpnsCacheSize + } + if cs < 0 { + return nil, fmt.Errorf("cannot specify negative resolve cache size") + } + + vsRouting = offlineroute.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator) + nsys, err = namesys.NewNameSystem(vsRouting, + namesys.WithDatastore(n.Repo.Datastore()), + namesys.WithDNSResolver(n.DNSResolver), + namesys.WithCache(cs)) + if err != nil { + return nil, fmt.Errorf("error constructing namesys: %w", err) + } } - offlineAPI, err := api.WithOptions(options.Api.Offline(true)) + + gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys)) if err != nil { return nil, err } + return &offlineGatewayErrWrapper{gwimpl: gw}, nil +} - return &gatewayAPI{ - ns: n.Namesys, - api: api, - offlineAPI: offlineAPI, - }, nil +type offlineGatewayErrWrapper struct { + gwimpl gateway.IPFSBackend } -func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Resolved) (files.Node, error) { - return gw.api.Unixfs().Get(ctx, pth) +func offlineErrWrap(err error) error { + if errors.Is(err, iface.ErrOffline) { + return fmt.Errorf("%s : %w", err.Error(), gateway.ErrServiceUnavailable) + } + return err } -func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Resolved) (<-chan iface.DirEntry, error) { - // Optimization: use Unixfs.Ls without resolving children, but using the - // cumulative DAG size as the file size. This allows for a fast listing - // while keeping a good enough Size field. - return gw.api.Unixfs().Ls(ctx, pth, - options.Unixfs.ResolveChildren(false), - options.Unixfs.UseCumulativeSize(true), - ) +func (o *offlineGatewayErrWrapper) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { + md, n, err := o.gwimpl.Get(ctx, path) + err = offlineErrWrap(err) + return md, n, err } -func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) { - r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid)) - if err != nil { - return nil, err - } +func (o *offlineGatewayErrWrapper) GetRange(ctx context.Context, path gateway.ImmutablePath, ranges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { + md, n, err := o.gwimpl.GetRange(ctx, path, ranges...) + err = offlineErrWrap(err) + return md, n, err +} - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } +func (o *offlineGatewayErrWrapper) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + md, n, err := o.gwimpl.GetAll(ctx, path) + err = offlineErrWrap(err) + return md, n, err +} - return blocks.NewBlockWithCid(data, cid) +func (o *offlineGatewayErrWrapper) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { + md, n, err := o.gwimpl.GetBlock(ctx, path) + err = offlineErrWrap(err) + return md, n, err } -func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - return gw.api.Routing().Get(ctx, "/ipns/"+c.String()) +func (o *offlineGatewayErrWrapper) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + md, n, err := o.gwimpl.Head(ctx, path) + err = offlineErrWrap(err) + return md, n, err } -func (gw *gatewayAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { - p, err := gw.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) - if err == namesys.ErrResolveRecursion { - err = nil - } - return path.New(p.String()), err +func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { + md, err := o.gwimpl.ResolvePath(ctx, path) + err = offlineErrWrap(err) + return md, err } -func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool { - _, err := gw.offlineAPI.Block().Stat(ctx, pth) - return err == nil +func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { + md, data, errCh, err := o.gwimpl.GetCAR(ctx, path) + err = offlineErrWrap(err) + return md, data, errCh, err } -func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) { - return gw.api.ResolvePath(ctx, pth) +func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool { + return o.gwimpl.IsCached(ctx, path) } +func (o *offlineGatewayErrWrapper) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + rec, err := o.gwimpl.GetIPNSRecord(ctx, c) + err = offlineErrWrap(err) + return rec, err +} + +func (o *offlineGatewayErrWrapper) ResolveMutable(ctx context.Context, path path.Path) (gateway.ImmutablePath, error) { + imPath, err := o.gwimpl.ResolveMutable(ctx, path) + err = offlineErrWrap(err) + return imPath, err +} + +func (o *offlineGatewayErrWrapper) GetDNSLinkRecord(ctx context.Context, s string) (path.Path, error) { + p, err := o.gwimpl.GetDNSLinkRecord(ctx, s) + err = offlineErrWrap(err) + return p, err +} + +var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil) + var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"} var subdomainGatewaySpec = &gateway.Specification{ diff --git a/core/node/dns.go b/core/node/dns.go index d66cccc5447..d338e0e8b67 100644 --- a/core/node/dns.go +++ b/core/node/dns.go @@ -1,88 +1,20 @@ package node import ( - "fmt" "math" - "strings" "time" + "github.com/ipfs/boxo/gateway" config "github.com/ipfs/kubo/config" doh "github.com/libp2p/go-doh-resolver" madns "github.com/multiformats/go-multiaddr-dns" - - "github.com/miekg/dns" ) -var defaultResolvers = map[string]string{ - "eth.": "https://resolver.cloudflare-eth.com/dns-query", - "crypto.": "https://resolver.cloudflare-eth.com/dns-query", -} - -func newResolver(url string, opts ...doh.Option) (madns.BasicResolver, error) { - if !strings.HasPrefix(url, "https://") { - return nil, fmt.Errorf("invalid resolver url: %s", url) - } - - return doh.NewResolver(url, opts...) -} - func DNSResolver(cfg *config.Config) (*madns.Resolver, error) { - var opts []madns.Option - var err error - var dohOpts []doh.Option if !cfg.DNS.MaxCacheTTL.IsDefault() { dohOpts = append(dohOpts, doh.WithMaxCacheTTL(cfg.DNS.MaxCacheTTL.WithDefault(time.Duration(math.MaxUint32)*time.Second))) } - domains := make(map[string]struct{}) // to track overridden default resolvers - rslvrs := make(map[string]madns.BasicResolver) // to reuse resolvers for the same URL - - for domain, url := range cfg.DNS.Resolvers { - if domain != "." && !dns.IsFqdn(domain) { - return nil, fmt.Errorf("invalid domain %s; must be FQDN", domain) - } - - domains[domain] = struct{}{} - if url == "" { - // allow overriding of implicit defaults with the default resolver - continue - } - - rslv, ok := rslvrs[url] - if !ok { - rslv, err = newResolver(url, dohOpts...) - if err != nil { - return nil, fmt.Errorf("bad resolver for %s: %w", domain, err) - } - rslvrs[url] = rslv - } - - if domain != "." { - opts = append(opts, madns.WithDomainResolver(domain, rslv)) - } else { - opts = append(opts, madns.WithDefaultResolver(rslv)) - } - } - - // fill in defaults if not overridden by the user - for domain, url := range defaultResolvers { - _, ok := domains[domain] - if ok { - continue - } - - rslv, ok := rslvrs[url] - if !ok { - rslv, err = newResolver(url) - if err != nil { - return nil, fmt.Errorf("bad resolver for %s: %w", domain, err) - } - rslvrs[url] = rslv - } - - opts = append(opts, madns.WithDomainResolver(domain, rslv)) - } - - return madns.NewResolver(opts...) + return gateway.NewDNSResolver(cfg.DNS.Resolvers, dohOpts...) } diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index bcbdf6d4bfd..06dbb08c1e6 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.0-rc2 + github.com/ipfs/boxo v0.8.0-rc3 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.26.4 github.com/multiformats/go-multiaddr v0.8.0 @@ -41,6 +41,7 @@ require ( github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect + github.com/gabriel-vasile/mimetype v1.4.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect @@ -77,6 +78,7 @@ require ( github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect + github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-ipld-format v0.4.0 // indirect @@ -153,6 +155,7 @@ require ( github.com/samber/lo v1.36.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 235e25c8316..8da997a7c5f 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -170,6 +170,8 @@ github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0X github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= +github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -335,8 +337,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0-rc2 h1:JnSlLKlIURiVsTfPs1BLv3NNCygjV1b07wGva3w4sLY= -github.com/ipfs/boxo v0.8.0-rc2/go.mod h1:EgDiNox/+W/+ySwEotRrHlvdmrhbSAB4p22ELg+ZsCc= +github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc= +github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -393,6 +395,8 @@ github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsb github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= +github.com/ipfs/go-ipfs-redirects-file v0.1.1 h1:Io++k0Vf/wK+tfnhEh63Yte1oQK5VGT2hIEYpD0Rzx8= +github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= @@ -800,8 +804,11 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -997,6 +1004,7 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1088,6 +1096,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= diff --git a/go.mod b/go.mod index 3648783d532..c3785381c6b 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.0-rc2 + github.com/ipfs/boxo v0.8.0-rc3 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-cidutil v0.1.0 @@ -55,7 +55,6 @@ require ( github.com/libp2p/go-libp2p-routing-helpers v0.6.1 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-socket-activation v0.1.0 - github.com/miekg/dns v1.1.50 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multiaddr v0.8.0 github.com/multiformats/go-multiaddr-dns v0.3.1 @@ -173,6 +172,7 @@ require ( github.com/mattn/go-runewidth v0.0.4 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect + github.com/miekg/dns v1.1.50 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.0 // indirect diff --git a/go.sum b/go.sum index 95daffbf755..5822ea8983c 100644 --- a/go.sum +++ b/go.sum @@ -355,8 +355,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0-rc2 h1:JnSlLKlIURiVsTfPs1BLv3NNCygjV1b07wGva3w4sLY= -github.com/ipfs/boxo v0.8.0-rc2/go.mod h1:EgDiNox/+W/+ySwEotRrHlvdmrhbSAB4p22ELg+ZsCc= +github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc= +github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/test/cli/fixtures/README.md b/test/cli/fixtures/README.md new file mode 100644 index 00000000000..dab926be480 --- /dev/null +++ b/test/cli/fixtures/README.md @@ -0,0 +1,72 @@ +# Dataset Description / Sources + +TestGatewayHAMTDirectory.car generated with: + +```bash +ipfs version +# ipfs version 0.19.0 + +export HAMT_DIR=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm +export IPFS_PATH=$(mktemp -d) + +# Init and start daemon, ensure we have an empty repository. +ipfs init --empty-repo +ipfs daemon &> /dev/null & +export IPFS_PID=$! + +# Retrieve the directory listing, forcing the daemon to download all required DAGs. Kill daemon. +curl -o dir.html http://127.0.0.1:8080/ipfs/$HAMT_DIR/ +kill $IPFS_PID + +# Get the list with all the downloaded refs and sanity check. +ipfs refs local > required_refs +cat required_refs | wc -l +# 962 + +# Get the list of all the files CIDs inside the directory and sanity check. +cat dir.html| pup '#content tbody .ipfs-hash attr{href}' | sed 's/\/ipfs\///g;s/\?filename=.*//g' > files_refs +cat files_refs | wc -l +# 10100 + +# Make and export our fixture. +ipfs files mkdir --cid-version 1 /fixtures +cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{} +cat files_refs | ipfs files write --create /fixtures/files_refs +export FIXTURE_CID=$(ipfs files stat --hash /fixtures/) +echo $FIXTURE_CID +# bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i +ipfs dag export $FIXTURE_CID > TestGatewayHAMTDirectory.car +``` + +TestGatewayMultiRange.car generated with: + + +```sh +ipfs version +# ipfs version 0.19.0 + +export FILE_CID=bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau +export IPFS_PATH=$(mktemp -d) + +# Init and start daemon, ensure we have an empty repository. +ipfs init --empty-repo +ipfs daemon &> /dev/null & +export IPFS_PID=$! + +# Get a specific byte range from the file. +curl http://127.0.0.1:8080/ipfs/$FILE_CID -i -H "Range: bytes=1276-1279, 29839070-29839080" +kill $IPFS_PID + +# Get the list with all the downloaded refs and sanity check. +ipfs refs local > required_refs +cat required_refs | wc -l +# 19 + +# Make and export our fixture. +ipfs files mkdir --cid-version 1 /fixtures +cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{} +export FIXTURE_CID=$(ipfs files stat --hash /fixtures/) +echo $FIXTURE_CID +# bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y +ipfs dag export $FIXTURE_CID > TestGatewayMultiRange.car +``` diff --git a/test/cli/fixtures/TestGatewayHAMTDirectory.car b/test/cli/fixtures/TestGatewayHAMTDirectory.car new file mode 100644 index 00000000000..2cb03c6c879 Binary files /dev/null and b/test/cli/fixtures/TestGatewayHAMTDirectory.car differ diff --git a/test/cli/fixtures/TestGatewayMultiRange.car b/test/cli/fixtures/TestGatewayMultiRange.car new file mode 100644 index 00000000000..993d86e33db Binary files /dev/null and b/test/cli/fixtures/TestGatewayMultiRange.car differ diff --git a/test/cli/gateway_range_test.go b/test/cli/gateway_range_test.go new file mode 100644 index 00000000000..5db7cc00e3e --- /dev/null +++ b/test/cli/gateway_range_test.go @@ -0,0 +1,75 @@ +package cli + +import ( + "fmt" + "net/http" + "os" + "testing" + + "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/assert" +) + +func TestGatewayHAMTDirectory(t *testing.T) { + t.Parallel() + + const ( + // The CID of the HAMT-sharded directory that has 10k items + hamtCid = "bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm" + + // fixtureCid is the CID of root of the DAG that is a subset of hamtCid DAG + // representing the minimal set of blocks necessary for directory listing. + // It also includes a "files_refs" file with the list of the references + // we do NOT needs to fetch (files inside the directory) + fixtureCid = "bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i" + ) + + // Start node + h := harness.NewT(t) + node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline") + client := node.GatewayClient() + + // Import fixtures + r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car") + assert.Nil(t, err) + defer r.Close() + err = node.IPFSDagImport(r, fixtureCid) + assert.Nil(t, err) + + // Fetch HAMT directory succeeds with minimal refs + resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid)) + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + +func TestGatewayMultiRange(t *testing.T) { + t.Parallel() + + const ( + // fileCid is the CID of the large HAMT-sharded file. + fileCid = "bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau" + + // fixtureCid is the CID of root of the DAG that is a subset of fileCid DAG + // representing the minimal set of blocks necessary for a simple byte range request. + fixtureCid = "bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y" + ) + + // Start node + h := harness.NewT(t) + node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline") + client := node.GatewayClient() + + // Import fixtures + r, err := os.Open("./fixtures/TestGatewayMultiRange.car") + assert.Nil(t, err) + defer r.Close() + err = node.IPFSDagImport(r, fixtureCid) + assert.Nil(t, err) + + // Succeeds fetching a range of blocks we have + resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) { + r.Header.Set("Range", "bytes=1276-1279, 29839070-29839080") + }) + assert.Equal(t, http.StatusPartialContent, resp.StatusCode) + assert.Contains(t, resp.Body, "Content-Range: bytes 1276-1279/109266405\r\nContent-Type: text/plain; charset=utf-8\r\n\r\niana\r\n") + assert.Contains(t, resp.Body, "Content-Range: bytes 29839070-29839080/109266405\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nEXAMPLE.COM\r\n") +} diff --git a/test/cli/harness/ipfs.go b/test/cli/harness/ipfs.go index 6ae7bdf947d..dde7e3495fc 100644 --- a/test/cli/harness/ipfs.go +++ b/test/cli/harness/ipfs.go @@ -78,3 +78,22 @@ func (n *Node) IPFSAdd(content io.Reader, args ...string) string { log.Debugf("add result: %q", out) return out } + +func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) error { + log.Debugf("node %d dag import with args: %v", n.ID, args) + fullArgs := []string{"dag", "import", "--pin-roots=false"} + fullArgs = append(fullArgs, args...) + res := n.Runner.MustRun(RunRequest{ + Path: n.IPFSBin, + Args: fullArgs, + CmdOpts: []CmdOpt{RunWithStdin(content)}, + }) + if res.Err != nil { + return res.Err + } + res = n.Runner.MustRun(RunRequest{ + Path: n.IPFSBin, + Args: []string{"block", "stat", "--offline", cid}, + }) + return res.Err +} diff --git a/test/sharness/t0117-gateway-block.sh b/test/sharness/t0117-gateway-block.sh index d5e40bb830b..b21425bc42c 100755 --- a/test/sharness/t0117-gateway-block.sh +++ b/test/sharness/t0117-gateway-block.sh @@ -29,6 +29,21 @@ FILE_CID=bafkreihhpc5y2pqvl5rbe5uuyhqjouybfs3rvlmisccgzue2kkt5zq6upq # ./dir/asc test_cmp expected_block curl_ipfs_dir_block_accept_output ' + test_expect_success "GET for application/vnd.ipld.raw with single range request includes correct bytes" ' + echo -n "application" > expected_file_block_single_range && + curl -sX GET -H "Accept: application/vnd.ipld.raw" -H "Range: bytes=6-16" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" -o curl_ipfs_file_block_single_range && + test_cmp expected_file_block_single_range curl_ipfs_file_block_single_range + ' + + test_expect_success "GET for application/vnd.ipld.raw with multiple range request includes correct bytes" ' + curl -sX GET -H "Accept: application/vnd.ipld.raw" -H "Range: bytes=6-16,0-4" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" -o curl_ipfs_file_block_multiple_range && + test_should_contain "Content-Range: bytes 6-16/31" curl_ipfs_file_block_multiple_range && + test_should_contain "Content-Type: application/vnd.ipld.raw" curl_ipfs_file_block_multiple_range && + test_should_contain "application" curl_ipfs_file_block_multiple_range && + test_should_contain "Content-Range: bytes 0-4/31" curl_ipfs_file_block_multiple_range && + test_should_contain "hello" curl_ipfs_file_block_multiple_range + ' + # Make sure expected HTTP headers are returned with the block bytes test_expect_success "GET response for application/vnd.ipld.raw has expected Content-Type" '