From f7aa1204b177231a3797576e1c01e560bacf66d9 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 30 Aug 2023 18:47:56 +0200 Subject: [PATCH 01/15] feat: add gateway to http over libp2p --- cmd/ipfs/daemon.go | 53 +++++++++++++++++++++++--- core/corehttp/commands.go | 4 +- core/corehttp/corehttp.go | 6 +-- core/corehttp/gateway.go | 27 ++++++++++++- core/corehttp/gateway_test.go | 2 +- docs/examples/kubo-as-a-library/go.mod | 4 +- docs/examples/kubo-as-a-library/go.sum | 8 ++-- go.mod | 4 +- go.sum | 8 ++-- test/dependencies/go.mod | 4 +- test/dependencies/go.sum | 8 ++-- 11 files changed, 96 insertions(+), 32 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 55ecaf2c09e..08f49d76cce 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -15,6 +15,9 @@ import ( multierror "github.com/hashicorp/go-multierror" + options "github.com/ipfs/boxo/coreiface/options" + cmds "github.com/ipfs/go-ipfs-cmds" + mprome "github.com/ipfs/go-metrics-prometheus" version "github.com/ipfs/kubo" utilmain "github.com/ipfs/kubo/cmd/ipfs/util" oldcmds "github.com/ipfs/kubo/commands" @@ -30,14 +33,12 @@ import ( fsrepo "github.com/ipfs/kubo/repo/fsrepo" "github.com/ipfs/kubo/repo/fsrepo/migrations" "github.com/ipfs/kubo/repo/fsrepo/migrations/ipfsfetcher" + goprocess "github.com/jbenet/goprocess" p2pcrypto "github.com/libp2p/go-libp2p/core/crypto" pnet "github.com/libp2p/go-libp2p/core/pnet" + "github.com/libp2p/go-libp2p/core/protocol" + p2phttp "github.com/libp2p/go-libp2p/p2p/http" sockets "github.com/libp2p/go-socket-activation" - - options "github.com/ipfs/boxo/coreiface/options" - cmds "github.com/ipfs/go-ipfs-cmds" - mprome "github.com/ipfs/go-metrics-prometheus" - goprocess "github.com/jbenet/goprocess" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" prometheus "github.com/prometheus/client_golang/prometheus" @@ -551,6 +552,12 @@ take effect. return err } + // add trustless gateway over libp2p + p2pGwErrc, err := serveTrustlessGatewayOverLibp2p(cctx) + if err != nil { + return err + } + // Add ipfs version info to prometheus metrics ipfsInfoMetric := promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "ipfs_info", @@ -617,7 +624,7 @@ take effect. // collect long-running errors and block for shutdown // TODO(cryptix): our fuse currently doesn't follow this pattern for graceful shutdown var errs error - for err := range merge(apiErrc, gwErrc, gcErrc) { + for err := range merge(apiErrc, gwErrc, gcErrc, p2pGwErrc) { if err != nil { errs = multierror.Append(errs, err) } @@ -899,6 +906,40 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e return errc, nil } +const gatewayProtocolID protocol.ID = "/ipfs-gateway" // FIXME: specify https://github.com/ipfs/specs/issues/433 + +func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error) { + opts := []corehttp.ServeOption{ + corehttp.MetricsCollectionOption("libp2p-gateway"), + corehttp.TrustlessGatewayOption(), + corehttp.VersionOption(), + } + + node, err := cctx.ConstructNode() + if err != nil { + return nil, fmt.Errorf("serveHTTPGateway: ConstructNode() failed: %s", err) + } + + handler, err := corehttp.MakeHandler(node, nil, opts...) + if err != nil { + return nil, err + } + + h := p2phttp.Host{ + StreamHost: node.PeerHost, + } + + h.SetHTTPHandler(gatewayProtocolID, handler) + + errc := make(chan error, 1) + go func() { + defer close(errc) + errc <- h.Serve() + }() + + return errc, nil +} + // collects options and opens the fuse mountpoint. func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error { cfg, err := cctx.GetConfig() diff --git a/core/corehttp/commands.go b/core/corehttp/commands.go index 63abc5922fc..53041e86bc5 100644 --- a/core/corehttp/commands.go +++ b/core/corehttp/commands.go @@ -166,7 +166,7 @@ func CommandsROOption(cctx oldcmds.Context) ServeOption { func CheckVersionOption() ServeOption { daemonVersion := version.ApiVersion - return ServeOption(func(n *core.IpfsNode, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) { + return func(n *core.IpfsNode, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) { mux := http.NewServeMux() parent.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, APIPath) { @@ -188,5 +188,5 @@ func CheckVersionOption() ServeOption { }) return mux, nil - }) + } } diff --git a/core/corehttp/corehttp.go b/core/corehttp/corehttp.go index b1a317f3c2a..6a9f43b5193 100644 --- a/core/corehttp/corehttp.go +++ b/core/corehttp/corehttp.go @@ -31,9 +31,9 @@ const shutdownTimeout = 30 * time.Second // initially passed in if not. type ServeOption func(*core.IpfsNode, net.Listener, *http.ServeMux) (*http.ServeMux, error) -// makeHandler turns a list of ServeOptions into a http.Handler that implements +// MakeHandler turns a list of ServeOptions into a http.Handler that implements // all of the given options, in order. -func makeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http.Handler, error) { +func MakeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http.Handler, error) { topMux := http.NewServeMux() mux := topMux for _, option := range options { @@ -86,7 +86,7 @@ func Serve(node *core.IpfsNode, lis net.Listener, options ...ServeOption) error // make sure we close this no matter what. defer lis.Close() - handler, err := makeHandler(node, lis, options...) + handler, err := MakeHandler(node, lis, options...) if err != nil { return err } diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index 105fa89af75..a954c92f30b 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -41,7 +41,7 @@ func GatewayOption(paths ...string) ServeOption { handler = otelhttp.NewHandler(handler, "Gateway") for _, p := range paths { - mux.HandleFunc(p+"/", handler.ServeHTTP) + mux.Handle(p+"/", handler) } return mux, nil @@ -61,7 +61,7 @@ func HostnameOption() ServeOption { } childMux := http.NewServeMux() - mux.HandleFunc("/", gateway.NewHostnameHandler(config, backend, childMux).ServeHTTP) + mux.Handle("/", gateway.NewHostnameHandler(config, backend, childMux)) return childMux, nil } } @@ -76,6 +76,29 @@ func VersionOption() ServeOption { } } +func TrustlessGatewayOption() ServeOption { + return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + config, err := getGatewayConfig(n) + if err != nil { + return nil, err + } + + bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore())) + + backend, err := gateway.NewBlocksBackend(bserv) + if err != nil { + return nil, err + } + + handler := gateway.NewHandler(config, &offlineGatewayErrWrapper{gwimpl: backend}) + handler = otelhttp.NewHandler(handler, "Libp2p-Gateway") + + mux.Handle("/ipfs/", handler) + + return mux, nil + } +} + func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) { cfg, err := n.Repo.Config() if err != nil { diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index 172988bba99..b3acda31ce2 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -121,7 +121,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface ts := httptest.NewServer(dh) t.Cleanup(func() { ts.Close() }) - dh.Handler, err = makeHandler(n, + dh.Handler, err = MakeHandler(n, ts.Listener, HostnameOption(), GatewayOption("/ipfs", "/ipns"), diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index bb88ad99696..ea374715a10 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.12.1-0.20230825151903-13569468babd github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 - github.com/libp2p/go-libp2p v0.30.0 + github.com/libp2p/go-libp2p v0.31.0 github.com/multiformats/go-multiaddr v0.11.0 ) @@ -148,7 +148,7 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.3.3 // indirect - github.com/quic-go/quic-go v0.38.0 // indirect + github.com/quic-go/quic-go v0.38.1 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/samber/lo v1.36.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 0ab47be1aef..d401c6faea1 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -458,8 +458,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U= -github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg= +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg= +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= @@ -640,8 +640,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM= github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= -github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc= -github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg= +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE= +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= diff --git a/go.mod b/go.mod index d3ff353c676..4150653e26e 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.30.0 + github.com/libp2p/go-libp2p v0.31.0 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.24.2 github.com/libp2p/go-libp2p-kbucket v0.6.3 @@ -192,7 +192,7 @@ require ( github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.3.3 // indirect - github.com/quic-go/quic-go v0.38.0 // indirect + github.com/quic-go/quic-go v0.38.1 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rs/cors v1.7.0 // indirect diff --git a/go.sum b/go.sum index e478ea73338..b116f9894c5 100644 --- a/go.sum +++ b/go.sum @@ -515,8 +515,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U= -github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg= +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg= +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= @@ -746,8 +746,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM= github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= -github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc= -github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg= +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE= +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 094cd9797b2..64a0f9bff88 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -19,7 +19,7 @@ require ( github.com/ipld/go-ipld-prime v0.21.0 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-random-files v0.0.0-20190219210431-31b3f20ebded - github.com/libp2p/go-libp2p v0.30.0 + github.com/libp2p/go-libp2p v0.31.0 github.com/multiformats/go-multiaddr v0.11.0 github.com/multiformats/go-multihash v0.2.3 gotest.tools/gotestsum v0.4.2 @@ -224,7 +224,7 @@ require ( github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.3.3 // indirect - github.com/quic-go/quic-go v0.38.0 // indirect + github.com/quic-go/quic-go v0.38.1 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 3c8cb6f87e3..89ae1027e3f 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -542,8 +542,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U= -github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg= +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg= +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= @@ -742,8 +742,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM= github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= -github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc= -github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg= +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE= +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= From a11c5424085771fde748702185a958d301ebb8f7 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 00:30:56 -0400 Subject: [PATCH 02/15] test(harness): skip environment variables starting with = --- test/cli/harness/run.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/cli/harness/run.go b/test/cli/harness/run.go index bec05fbce68..8ca85eb63b1 100644 --- a/test/cli/harness/run.go +++ b/test/cli/harness/run.go @@ -48,6 +48,11 @@ func environToMap(environ []string) map[string]string { m := map[string]string{} for _, e := range environ { kv := strings.Split(e, "=") + // Skip environment variables that start with = + // These can occur in Windows https://github.com/golang/go/issues/61956 + if kv[0] == "" { + continue + } m[kv[0]] = kv[1] } return m From fb5cacac86786597bd31d88867d5487ac0224b89 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 01:47:35 -0400 Subject: [PATCH 03/15] test(harness): use SIGKILL to terminate daemons in Windows testing --- test/cli/harness/node.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/cli/harness/node.go b/test/cli/harness/node.go index 9fc01f4d7ca..7db1d553824 100644 --- a/test/cli/harness/node.go +++ b/test/cli/harness/node.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strconv" "strings" "syscall" @@ -278,6 +279,15 @@ func (n *Node) StopDaemon() *Node { _, _ = n.Daemon.Cmd.Process.Wait() watch <- struct{}{} }() + + // os.Interrupt does not support interrupts on Windows https://github.com/golang/go/issues/46345 + if runtime.GOOS == "windows" { + if n.signalAndWait(watch, syscall.SIGKILL, 5*time.Second) { + return n + } + log.Panicf("timed out stopping node %d with peer ID %s", n.ID, n.PeerID()) + } + log.Debugf("signaling node %d with SIGTERM", n.ID) if n.signalAndWait(watch, syscall.SIGTERM, 1*time.Second) { return n From 5b0da083330841bc031f410b4e503ccc12d9cd49 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 03:48:23 -0400 Subject: [PATCH 04/15] fix: use /ipfs/gateway as the protocol ID for serving the gateway over libp2p --- cmd/ipfs/daemon.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 08f49d76cce..ed8040c01dc 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -906,7 +906,7 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e return errc, nil } -const gatewayProtocolID protocol.ID = "/ipfs-gateway" // FIXME: specify https://github.com/ipfs/specs/issues/433 +const gatewayProtocolID protocol.ID = "/ipfs/gateway" // FIXME: specify https://github.com/ipfs/specs/issues/433 func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error) { opts := []corehttp.ServeOption{ From 9d32f71e22299604ba7748549eaad4e667198712 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 03:49:35 -0400 Subject: [PATCH 05/15] move the gateway-over-libp2p mountpoint to the root --- cmd/ipfs/daemon.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index ed8040c01dc..b28b5976471 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -929,7 +929,13 @@ func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error StreamHost: node.PeerHost, } - h.SetHTTPHandler(gatewayProtocolID, handler) + tmpProtocol := protocol.ID("/kubo/delete-me") + h.SetHTTPHandler(tmpProtocol, http.NotFoundHandler()) + h.WellKnownHandler.RemoveProtocolMeta(tmpProtocol) + + h.WellKnownHandler.AddProtocolMeta(gatewayProtocolID, p2phttp.ProtocolMeta{Path: "/"}) + h.ServeMux = http.NewServeMux() + h.ServeMux.Handle("/", handler) errc := make(chan error, 1) go func() { From bc6bee15857b3ca24d319c2d8eb62b75a245b0df Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 03:51:48 -0400 Subject: [PATCH 06/15] rename Libp2pGatewayOption and hard code its gateway configuration --- cmd/ipfs/daemon.go | 2 +- core/corehttp/gateway.go | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index b28b5976471..f11d8695270 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -911,7 +911,7 @@ const gatewayProtocolID protocol.ID = "/ipfs/gateway" // FIXME: specify https:// func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error) { opts := []corehttp.ServeOption{ corehttp.MetricsCollectionOption("libp2p-gateway"), - corehttp.TrustlessGatewayOption(), + corehttp.Libp2pGatewayOption(), corehttp.VersionOption(), } diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index a954c92f30b..4a4f087809c 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -76,13 +76,8 @@ func VersionOption() ServeOption { } } -func TrustlessGatewayOption() ServeOption { +func Libp2pGatewayOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - config, err := getGatewayConfig(n) - if err != nil { - return nil, err - } - bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore())) backend, err := gateway.NewBlocksBackend(bserv) @@ -90,7 +85,14 @@ func TrustlessGatewayOption() ServeOption { return nil, err } - handler := gateway.NewHandler(config, &offlineGatewayErrWrapper{gwimpl: backend}) + gwConfig := gateway.Config{ + DeserializedResponses: false, + NoDNSLink: true, + PublicGateways: nil, + Menu: nil, + } + + handler := gateway.NewHandler(gwConfig, &offlineGatewayErrWrapper{gwimpl: backend}) handler = otelhttp.NewHandler(handler, "Libp2p-Gateway") mux.Handle("/ipfs/", handler) From bf548a3cc625438828aaae9a081b0a97dfb0e821 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 04:07:10 -0400 Subject: [PATCH 07/15] fix(gateway): close http-over-libp2p host when the node is ready to shutdown --- cmd/ipfs/daemon.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index f11d8695270..0763f6eb28b 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -943,6 +943,11 @@ func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error errc <- h.Serve() }() + go func() { + <-node.Process.Closing() + h.Close() + }() + return errc, nil } From 2a1d91f67abda10c02072d14539940b777df500b Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 03:52:51 -0400 Subject: [PATCH 08/15] test(gateway): add harness tests for gateway over libp2p --- test/cli/http_gateway_over_libp2p_test.go | 106 ++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 test/cli/http_gateway_over_libp2p_test.go diff --git a/test/cli/http_gateway_over_libp2p_test.go b/test/cli/http_gateway_over_libp2p_test.go new file mode 100644 index 00000000000..b4b2cf1056d --- /dev/null +++ b/test/cli/http_gateway_over_libp2p_test.go @@ -0,0 +1,106 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "testing" + + "github.com/ipfs/go-cid" + "github.com/ipfs/kubo/core/commands" + "github.com/ipfs/kubo/test/cli/harness" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/peer" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" + "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/stretchr/testify/require" +) + +func TestGatewayOverLibp2p(t *testing.T) { + t.Parallel() + nodes := harness.NewT(t).NewNodes(2).Init() + + // Setup streaming functionality + nodes.ForEachPar(func(node *harness.Node) { + node.IPFS("config", "--json", "Experimental.Libp2pStreamMounting", "true") + }) + + gwNode := nodes[0] + p2pProxyNode := nodes[1] + + nodes.StartDaemons().Connect() + + // Add data to the gateway node + cidDataOnGatewayNode := cid.MustParse(gwNode.IPFSAddStr("Hello Worlds2!")) + r := gwNode.GatewayClient().Get(fmt.Sprintf("/ipfs/%s?format=raw", cidDataOnGatewayNode)) + blockDataOnGatewayNode := []byte(r.Body) + + // Add data to the non-gateway node + cidDataNotOnGatewayNode := cid.MustParse(p2pProxyNode.IPFSAddStr("Hello Worlds!")) + r = p2pProxyNode.GatewayClient().Get(fmt.Sprintf("/ipfs/%s?format=raw", cidDataNotOnGatewayNode)) + blockDataNotOnGatewayNode := []byte(r.Body) + _ = blockDataNotOnGatewayNode + + // Setup one of the nodes as http to http-over-libp2p proxy + p2pProxyNode.IPFS("p2p", "forward", "--allow-custom-protocol", "/http/1.1", "/ip4/127.0.0.1/tcp/0", fmt.Sprintf("/p2p/%s", gwNode.PeerID())) + lsOutput := commands.P2PLsOutput{} + if err := json.Unmarshal(p2pProxyNode.IPFS("p2p", "ls", "--enc=json").Stdout.Bytes(), &lsOutput); err != nil { + t.Fatal(err) + } + require.Len(t, lsOutput.Listeners, 1) + p2pProxyNodeHTTPListenMA, err := multiaddr.NewMultiaddr(lsOutput.Listeners[0].ListenAddress) + require.NoError(t, err) + + p2pProxyNodeHTTPListenAddr, err := manet.ToNetAddr(p2pProxyNodeHTTPListenMA) + require.NoError(t, err) + + // Note: the bare HTTP requests here assume that the gateway is mounted at `/` + t.Run("WillNotServeRemoteContent", func(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s?format=raw", p2pProxyNodeHTTPListenAddr, cidDataNotOnGatewayNode)) + require.NoError(t, err) + require.Equal(t, 500, resp.StatusCode) + }) + + t.Run("WillNotServeDeserializedResponses", func(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s", p2pProxyNodeHTTPListenAddr, cidDataOnGatewayNode)) + require.NoError(t, err) + require.Equal(t, http.StatusNotAcceptable, resp.StatusCode) + }) + + t.Run("ServeBlock", func(t *testing.T) { + t.Run("UsingKuboProxy", func(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s?format=raw", p2pProxyNodeHTTPListenAddr, cidDataOnGatewayNode)) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, blockDataOnGatewayNode, body) + }) + t.Run("UsingLibp2pClientWithPathDiscovery", func(t *testing.T) { + clientHost, err := libp2p.New(libp2p.NoListenAddrs) + require.NoError(t, err) + err = clientHost.Connect(context.Background(), peer.AddrInfo{ + ID: gwNode.PeerID(), + Addrs: gwNode.SwarmAddrs(), + }) + require.NoError(t, err) + + client, err := (&libp2phttp.Host{StreamHost: clientHost}).NamespacedClient("/ipfs/gateway", peer.AddrInfo{ID: gwNode.PeerID()}) + require.NoError(t, err) + + resp, err := client.Get(fmt.Sprintf("/ipfs/%s?format=raw", cidDataOnGatewayNode)) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + require.Equal(t, blockDataOnGatewayNode, body) + }) + }) +} From b8c741d8bf18080e1f29c3dadd384b580953ce83 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 03:53:48 -0400 Subject: [PATCH 09/15] test(gateway-conformance): also run gateway conformance tests against a gateway-over-libp2p endpoint --- .github/workflows/gateway-conformance.yml | 104 +++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 928d51c7007..a1b96402ac9 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -58,8 +58,10 @@ jobs: } } run: | - ./ipfs init + ./ipfs init --profile=test ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" + ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" + ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" working-directory: kubo-gateway/cmd/ipfs # 4. Populate the Kubo gateway with the gateway-conformance fixtures @@ -85,7 +87,28 @@ jobs: # 5. Start the kubo-gateway - name: Start kubo-gateway run: | - ./ipfs daemon --offline & + ./ipfs daemon & + endpoint="http://127.0.0.1:5001/api/v0/version" + max_retries=5 + retry_interval=3 + + check_endpoint() { + curl -X POST --silent --fail "$endpoint" > /dev/null + return $? + } + + retries=0 + while ! check_endpoint; do + retries=$((retries+1)) + + if [ $retries -ge $max_retries ]; then + echo "daemon took too long to start" + exit 1 + fi + + sleep $retry_interval + done + echo "daemon started and ready to receive API calls" working-directory: kubo-gateway/cmd/ipfs # 6. Run the gateway-conformance tests @@ -115,3 +138,80 @@ jobs: with: name: gateway-conformance.json path: output.json + + # 8. Setup a kubo http-p2p-proxy to run gateway conformance tests + - name: Init p2p-proxy kubo node + env: + IPFS_PATH: "~/.kubo-p2p-proxy" + run: | + ./ipfs init --profile=test + ./ipfs config --json Experimental.Libp2pStreamMounting true + ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081" + ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002" + working-directory: kubo-gateway/cmd/ipfs + + # 9. Start the kubo http-p2p-proxy + - name: Start kubo http-p2p-proxy + env: + IPFS_PATH: "~/.kubo-p2p-proxy" + run: | + ./ipfs daemon & + + endpoint="http://127.0.0.1:5002/api/v0/version" + max_retries=5 + retry_interval=3 + + check_endpoint() { + curl -X POST --silent --fail "$endpoint" > /dev/null + return $? + } + + retries=0 + while ! check_endpoint; do + retries=$((retries+1)) + + if [ $retries -ge $max_retries ]; then + echo "daemon took too long to start" + exit 1 + fi + + sleep $retry_interval + done + echo "daemon started and ready to receive API calls" + working-directory: kubo-gateway/cmd/ipfs + + # 10. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p + - name: Start http-over-libp2p forwarding proxy + run: | + gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="") + ./ipfs --api=/ip4/127.0.0.1/tcp/5002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1) + ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8082 /p2p/$gatewayNodeId + working-directory: kubo-gateway/cmd/ipfs + + # 11. Run the gateway-conformance tests over libp2p + - name: Run gateway-conformance tests over libp2p + uses: ipfs/gateway-conformance/.github/actions/test@v0.3 + with: + gateway-url: http://127.0.0.1:8081 + json: output.json + xml: output.xml + html: output.html + markdown: output.md + args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' + + # 11. Upload the results + - name: Upload MD summary + if: failure() || success() + run: cat output.md >> $GITHUB_STEP_SUMMARY + - name: Upload HTML report + if: failure() || success() + uses: actions/upload-artifact@v3 + with: + name: gateway-conformance-libp2p.html + path: output.html + - name: Upload JSON report + if: failure() || success() + uses: actions/upload-artifact@v3 + with: + name: gateway-conformance-libp2p.json + path: output.json \ No newline at end of file From f2c43d5bbfb4efec8e32573eff929feed472e734 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 05:22:51 -0400 Subject: [PATCH 10/15] feat(config): Add gateway-over-libp2p experiment --- .github/workflows/gateway-conformance.yml | 1 + cmd/ipfs/daemon.go | 20 +++++++++--- config/experiments.go | 1 + docs/experimental-features.md | 37 +++++++++++++++++++++++ test/cli/http_gateway_over_libp2p_test.go | 10 ++++++ 5 files changed, 64 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index a1b96402ac9..d49291a43e3 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -60,6 +60,7 @@ jobs: run: | ./ipfs init --profile=test ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" + ./ipfs config --json Experimental.GatewayOverLibp2p true ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" working-directory: kubo-gateway/cmd/ipfs diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 0763f6eb28b..d9207b0f61a 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -909,17 +909,27 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e const gatewayProtocolID protocol.ID = "/ipfs/gateway" // FIXME: specify https://github.com/ipfs/specs/issues/433 func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error) { + node, err := cctx.ConstructNode() + if err != nil { + return nil, fmt.Errorf("serveHTTPGatewayOverLibp2p: ConstructNode() failed: %s", err) + } + cfg, err := node.Repo.Config() + if err != nil { + return nil, fmt.Errorf("could not read config: %w", err) + } + + if !cfg.Experimental.GatewayOverLibp2p { + errCh := make(chan error) + close(errCh) + return errCh, nil + } + opts := []corehttp.ServeOption{ corehttp.MetricsCollectionOption("libp2p-gateway"), corehttp.Libp2pGatewayOption(), corehttp.VersionOption(), } - node, err := cctx.ConstructNode() - if err != nil { - return nil, fmt.Errorf("serveHTTPGateway: ConstructNode() failed: %s", err) - } - handler, err := corehttp.MakeHandler(node, nil, opts...) if err != nil { return nil, err diff --git a/config/experiments.go b/config/experiments.go index f5ecf4be629..3a63d253da4 100644 --- a/config/experiments.go +++ b/config/experiments.go @@ -11,4 +11,5 @@ type Experiments struct { AcceleratedDHTClient experimentalAcceleratedDHTClient `json:",omitempty"` OptimisticProvide bool OptimisticProvideJobsPoolSize int + GatewayOverLibp2p bool `json:",omitempty"` } diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 587d136d506..52bfe703dbf 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -27,6 +27,7 @@ the above issue. - [Graphsync](#graphsync) - [Noise](#noise) - [Optimistic Provide](#optimistic-provide) +- [HTTP Gateway over Libp2p](#http-gateway-over-libp2p) --- @@ -617,3 +618,39 @@ ipfs config --json Experimental.OptimisticProvideJobsPoolSize 120 - [ ] Needs more people to use and report on how well it works - [ ] Should prove at least equivalent availability of provider records as the classic approach + +## HTTP Gateway over Libp2p + +### In Version + +0.23.0 + +### State + +Experimental, disabled by default. + +Enables serving the [IPFS HTTP Gateway](https://specs.ipfs.tech/http-gateways/) protocol over libp2p transports and +as described in the [specification](https://github.com/ipfs/specs/pull/434). + +Notes: +- This feature currently is only about serving the gateway requests over libp2p, not about fetching data this way using +[Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). +- While kubo currently mounts the gateway API at the root (i.e. `/`) of the libp2p `/http/1.1` protocol that is subject to +change. The way to reliably discover where a given HTTP protocol is mounted on a libp2p endpoint is via the `.well-known/libp2p` +resource specified in the [http+libp2p specification](https://github.com/libp2p/specs/pull/508) +- Kubo currently hard codes the gateway-over-libp2p behavior to: + - Only operate on `/ipfs` resources + - Only satisfy the Trustless Gateway API + - Only serve data that is already local to the node (i.e. similar to a `NoFetch` gateway) + +### How to enable + +Modify your ipfs config: + +``` +ipfs config --json Experimental.GatewayOverLibp2p true +``` + +### Road to being a real feature + +- [ ] Needs more people to use and report on how well it works \ No newline at end of file diff --git a/test/cli/http_gateway_over_libp2p_test.go b/test/cli/http_gateway_over_libp2p_test.go index b4b2cf1056d..ee57175719d 100644 --- a/test/cli/http_gateway_over_libp2p_test.go +++ b/test/cli/http_gateway_over_libp2p_test.go @@ -57,6 +57,16 @@ func TestGatewayOverLibp2p(t *testing.T) { p2pProxyNodeHTTPListenAddr, err := manet.ToNetAddr(p2pProxyNodeHTTPListenMA) require.NoError(t, err) + t.Run("DoesNotWorkWithoutExperimentalConfig", func(t *testing.T) { + _, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s?format=raw", p2pProxyNodeHTTPListenAddr, cidDataOnGatewayNode)) + require.Error(t, err) + }) + + // Enable the experimental feature and reconnect the nodes + gwNode.IPFS("config", "--json", "Experimental.GatewayOverLibp2p", "true") + gwNode.StopDaemon().StartDaemon() + nodes.Connect() + // Note: the bare HTTP requests here assume that the gateway is mounted at `/` t.Run("WillNotServeRemoteContent", func(t *testing.T) { resp, err := http.Get(fmt.Sprintf("http://%s/ipfs/%s?format=raw", p2pProxyNodeHTTPListenAddr, cidDataNotOnGatewayNode)) From d86192a9bf3d5336298a4c08a62b8656addb6600 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 31 Aug 2023 05:23:39 -0400 Subject: [PATCH 11/15] changelog(gateway-over-libp2p): add gateway-over-libp2p changelog --- docs/changelogs/v0.23.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/changelogs/v0.23.md b/docs/changelogs/v0.23.md index df6a1c0316e..701f0570123 100644 --- a/docs/changelogs/v0.23.md +++ b/docs/changelogs/v0.23.md @@ -10,6 +10,7 @@ - [Gateway: meaningful CAR responses on Not Found errors](#gateway-meaningful-car-responses-on-not-found-errors) - [Binary characters in file names: no longer works with old clients and new Kubo servers](#binary-characters-in-file-names-no-longer-works-with-old-clients-and-new-kubo-servers) - [Self-hosting `/routing/v1` endpoint for delegated routing needs](#self-hosting-routingv1-endpoint-for-delegated-routing-needs) + - [Gateway Over Libp2p Experiment](#gateway-over-libp2p-experiment) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -67,6 +68,20 @@ HTTP [Routing V1](https://specs.ipfs.tech/routing/http-routing-v1/) API endpoint self-hosting and experimentation with custom delegated routers. This is disabled by default, but can be enabled by setting [`Gateway.ExposeRoutingAPI`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayexposeroutingapi) to `true` . +#### Gateway Over Libp2p Experiment + +It is now possible to serve [Trustless Gateway API](https://specs.ipfs.tech/http-gateways/trustless-gateway/) responses +such as for blocks and CARs over libp2p. This takes advantage of the [specification work](https://github.com/libp2p/specs/pull/508) +in libp2p expanding beyond the basics of performing HTTP requests over libp2p streams that have been available in libp2p +and in kubo experimental features such as [p2p-http-proxy](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#p2p-http-proxy). + +This means that implementations that want to use the Trustless Gateway API as a data transport mechanism can do so even +when standard HTTP transports would fail (e.g. when the endpoint is behind a firewall, or wants to serve data to a browser +but does not have a CA certificate). + +See [HTTP Gateway over Libp2p](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p) +for more details. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors From df66dacc263c79b5bae54f03cdea5eb16388fa54 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 1 Sep 2023 14:28:35 -0400 Subject: [PATCH 12/15] test(sharness): update ping test since go-libp2p v0.31.0 changed the reported error message --- test/sharness/t0041-ping.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sharness/t0041-ping.sh b/test/sharness/t0041-ping.sh index 4bbf89969ff..6a817060a7b 100755 --- a/test/sharness/t0041-ping.sh +++ b/test/sharness/t0041-ping.sh @@ -27,7 +27,7 @@ test_expect_success "test ping other" ' test_expect_success "test ping unreachable peer" ' printf "Looking up peer %s\n" "$BAD_PEER" > bad_ping_exp && - printf "PING QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJx.\nPing error: failed to dial: no addresses\nError: ping failed\n" >> bad_ping_exp && + printf "PING QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJx.\nPing error: failed to dial: failed to dial QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJx: no addresses\nError: ping failed\n" >> bad_ping_exp && ! ipfsi 0 ping -n2 -- "$BAD_PEER" > bad_ping_actual 2>&1 && test_cmp bad_ping_exp bad_ping_actual ' From 1efd9d47aa6ff05072bdfc9115a02797a4d50e40 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 6 Sep 2023 01:33:39 +0200 Subject: [PATCH 13/15] refactor(ci): libp2p conformance is separate job this ensures the libp2p experiment runs independently and its failure does not impact the result of job that tests stable features on http port --- .github/workflows/gateway-conformance.yml | 155 ++++++++++++---------- 1 file changed, 83 insertions(+), 72 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index d49291a43e3..06e2701d24d 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -17,7 +17,24 @@ defaults: run: shell: bash +env: + # hostnames expected by https://github.com/ipfs/gateway-conformance + GATEWAY_PUBLIC_GATEWAYS: | + { + "example.com": { + "UseSubdomains": true, + "InlineDNSLink": true, + "Paths": ["/ipfs", "/ipns"] + }, + "localhost": { + "UseSubdomains": true, + "InlineDNSLink": true, + "Paths": ["/ipfs", "/ipns"] + } + } + jobs: + # Testing all gateway features via TCP port specified in Addresses.Gateway gateway-conformance: runs-on: ubuntu-latest timeout-minutes: 10 @@ -30,11 +47,14 @@ jobs: # 2. Build the kubo-gateway - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.20.x + - uses: protocol/cache-go-action@v1 + with: + name: ${{ github.job }} - name: Checkout kubo-gateway - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: kubo-gateway - name: Build kubo-gateway @@ -43,26 +63,9 @@ jobs: # 3. Init the kubo-gateway - name: Init kubo-gateway - env: - GATEWAY_PUBLIC_GATEWAYS: | - { - "example.com": { - "UseSubdomains": true, - "InlineDNSLink": true, - "Paths": ["/ipfs", "/ipns"] - }, - "localhost": { - "UseSubdomains": true, - "InlineDNSLink": true, - "Paths": ["/ipfs", "/ipns"] - } - } run: | - ./ipfs init --profile=test + ./ipfs init -e ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" - ./ipfs config --json Experimental.GatewayOverLibp2p true - ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" - ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" working-directory: kubo-gateway/cmd/ipfs # 4. Populate the Kubo gateway with the gateway-conformance fixtures @@ -88,28 +91,7 @@ jobs: # 5. Start the kubo-gateway - name: Start kubo-gateway run: | - ./ipfs daemon & - endpoint="http://127.0.0.1:5001/api/v0/version" - max_retries=5 - retry_interval=3 - - check_endpoint() { - curl -X POST --silent --fail "$endpoint" > /dev/null - return $? - } - - retries=0 - while ! check_endpoint; do - retries=$((retries+1)) - - if [ $retries -ge $max_retries ]; then - echo "daemon took too long to start" - exit 1 - fi - - sleep $retry_interval - done - echo "daemon started and ready to receive API calls" + ./ipfs daemon --offline & working-directory: kubo-gateway/cmd/ipfs # 6. Run the gateway-conformance tests @@ -140,48 +122,77 @@ jobs: name: gateway-conformance.json path: output.json - # 8. Setup a kubo http-p2p-proxy to run gateway conformance tests + # Testing trustless gateway feature subset exposed as libp2p protocol + gateway-conformance-libp2p-experiment: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # 1. Download the gateway-conformance fixtures + - name: Download gateway-conformance fixtures + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3 + with: + output: fixtures + + # 2. Build the kubo-gateway + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + - uses: protocol/cache-go-action@v1 + with: + name: ${{ github.job }} + - name: Checkout kubo-gateway + uses: actions/checkout@v3 + with: + path: kubo-gateway + - name: Build kubo-gateway + run: make build + working-directory: kubo-gateway + + # 3. Init the kubo-gateway + - name: Init kubo-gateway + run: | + ./ipfs init --profile=test + ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS" + ./ipfs config --json Experimental.GatewayOverLibp2p true + ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080" + ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001" + working-directory: kubo-gateway/cmd/ipfs + + # 4. Populate the Kubo gateway with the gateway-conformance fixtures + - name: Import fixtures + run: | + # Import car files + find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \; + + # 5. Start the kubo-gateway + - name: Start kubo-gateway + run: | + ( ./ipfs daemon & ) | sed '/Daemon is ready/q' + while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done + working-directory: kubo-gateway/cmd/ipfs + + # 6. Setup a kubo http-p2p-proxy to expose libp2p protocol as a regular HTTP port for gateway conformance tests - name: Init p2p-proxy kubo node env: IPFS_PATH: "~/.kubo-p2p-proxy" run: | - ./ipfs init --profile=test + ./ipfs init --profile=test -e ./ipfs config --json Experimental.Libp2pStreamMounting true ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081" ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002" working-directory: kubo-gateway/cmd/ipfs - # 9. Start the kubo http-p2p-proxy + # 7. Start the kubo http-p2p-proxy - name: Start kubo http-p2p-proxy env: IPFS_PATH: "~/.kubo-p2p-proxy" run: | - ./ipfs daemon & - - endpoint="http://127.0.0.1:5002/api/v0/version" - max_retries=5 - retry_interval=3 - - check_endpoint() { - curl -X POST --silent --fail "$endpoint" > /dev/null - return $? - } - - retries=0 - while ! check_endpoint; do - retries=$((retries+1)) - - if [ $retries -ge $max_retries ]; then - echo "daemon took too long to start" - exit 1 - fi - - sleep $retry_interval - done - echo "daemon started and ready to receive API calls" + ( ./ipfs daemon & ) | sed '/Daemon is ready/q' + while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done working-directory: kubo-gateway/cmd/ipfs - # 10. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p + # 8. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p - name: Start http-over-libp2p forwarding proxy run: | gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="") @@ -189,7 +200,7 @@ jobs: ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8082 /p2p/$gatewayNodeId working-directory: kubo-gateway/cmd/ipfs - # 11. Run the gateway-conformance tests over libp2p + # 9. Run the gateway-conformance tests over libp2p - name: Run gateway-conformance tests over libp2p uses: ipfs/gateway-conformance/.github/actions/test@v0.3 with: @@ -200,7 +211,7 @@ jobs: markdown: output.md args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length' - # 11. Upload the results + # 10. Upload the results - name: Upload MD summary if: failure() || success() run: cat output.md >> $GITHUB_STEP_SUMMARY @@ -215,4 +226,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: gateway-conformance-libp2p.json - path: output.json \ No newline at end of file + path: output.json From 8d28507814021d9705ea9d2d5480f6e580237cf6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 6 Sep 2023 02:57:14 +0200 Subject: [PATCH 14/15] docs: gateway-http-over-libp2p --- docs/changelogs/v0.23.md | 23 ++++++++++++----------- docs/experimental-features.md | 33 +++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/changelogs/v0.23.md b/docs/changelogs/v0.23.md index 701f0570123..6085c21c788 100644 --- a/docs/changelogs/v0.23.md +++ b/docs/changelogs/v0.23.md @@ -10,7 +10,7 @@ - [Gateway: meaningful CAR responses on Not Found errors](#gateway-meaningful-car-responses-on-not-found-errors) - [Binary characters in file names: no longer works with old clients and new Kubo servers](#binary-characters-in-file-names-no-longer-works-with-old-clients-and-new-kubo-servers) - [Self-hosting `/routing/v1` endpoint for delegated routing needs](#self-hosting-routingv1-endpoint-for-delegated-routing-needs) - - [Gateway Over Libp2p Experiment](#gateway-over-libp2p-experiment) + - [Trustless Gateway Over Libp2p Experiment](#trustless-gateway-over-libp2p-experiment) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -68,19 +68,20 @@ HTTP [Routing V1](https://specs.ipfs.tech/routing/http-routing-v1/) API endpoint self-hosting and experimentation with custom delegated routers. This is disabled by default, but can be enabled by setting [`Gateway.ExposeRoutingAPI`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayexposeroutingapi) to `true` . -#### Gateway Over Libp2p Experiment +#### Trustless Gateway Over Libp2p Experiment -It is now possible to serve [Trustless Gateway API](https://specs.ipfs.tech/http-gateways/trustless-gateway/) responses -such as for blocks and CARs over libp2p. This takes advantage of the [specification work](https://github.com/libp2p/specs/pull/508) -in libp2p expanding beyond the basics of performing HTTP requests over libp2p streams that have been available in libp2p -and in kubo experimental features such as [p2p-http-proxy](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#p2p-http-proxy). +In this update, we've introduced an experimental opt-in feature allowing users to +serve a subset of [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) responses, +such as blocks and CARs, over libp2p. This enhancement leverages the ongoing +[`/http/1.1` specification work in libp2p](https://github.com/libp2p/specs/pull/508) +to make it easier to support HTTP semantics over libp2p streams. -This means that implementations that want to use the Trustless Gateway API as a data transport mechanism can do so even -when standard HTTP transports would fail (e.g. when the endpoint is behind a firewall, or wants to serve data to a browser -but does not have a CA certificate). +This development means that if users wish to utilize the Trustless Gateway API +for data transport, they can now do so even in scenarios where standard HTTP +might be problematic, such as when the endpoint is behind a firewall or when +attempting to serve data to a browser without a CA certificate. -See [HTTP Gateway over Libp2p](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p) -for more details. +See [HTTP Gateway over Libp2p](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p) for details about this experiment. ### ๐Ÿ“ Changelog diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 52bfe703dbf..6527f883b58 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -629,19 +629,25 @@ ipfs config --json Experimental.OptimisticProvideJobsPoolSize 120 Experimental, disabled by default. -Enables serving the [IPFS HTTP Gateway](https://specs.ipfs.tech/http-gateways/) protocol over libp2p transports and -as described in the [specification](https://github.com/ipfs/specs/pull/434). +Enables serving a subset of the [IPFS HTTP Gateway](https://specs.ipfs.tech/http-gateways/) semantics over libp2p `/http/1.1` protocol. Notes: -- This feature currently is only about serving the gateway requests over libp2p, not about fetching data this way using -[Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). -- While kubo currently mounts the gateway API at the root (i.e. `/`) of the libp2p `/http/1.1` protocol that is subject to -change. The way to reliably discover where a given HTTP protocol is mounted on a libp2p endpoint is via the `.well-known/libp2p` -resource specified in the [http+libp2p specification](https://github.com/libp2p/specs/pull/508) -- Kubo currently hard codes the gateway-over-libp2p behavior to: - - Only operate on `/ipfs` resources - - Only satisfy the Trustless Gateway API - - Only serve data that is already local to the node (i.e. similar to a `NoFetch` gateway) +- This feature only about serving verifiable gateway requests over libp2p: + - Deserialized responses are not supported. + - Only operate on `/ipfs` resources (no `/ipns` atm) + - Only support requests for `application/vnd.ipld.raw` and + `application/vnd.ipld.car` (from [Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/), + where data integrity can be verified). + - Only serve data that is already local to the node (i.e. similar to a + [`Gateway.NoFetch`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaynofetch)) +- While Kubo currently mounts the gateway API at the root (i.e. `/`) of the + libp2p `/http/1.1` protocol, that is subject to change. + - The way to reliably discover where a given HTTP protocol is mounted on a + libp2p endpoint is via the `.well-known/libp2p` resource specified in the + [http+libp2p specification](https://github.com/libp2p/specs/pull/508) + - The identifier of the protocol mount point under `/http/1.1` listener is + `/ipfs/gateway`, as noted in + [ipfs/specs#434](https://github.com/ipfs/specs/pull/434). ### How to enable @@ -653,4 +659,7 @@ ipfs config --json Experimental.GatewayOverLibp2p true ### Road to being a real feature -- [ ] Needs more people to use and report on how well it works \ No newline at end of file +- [ ] Needs more people to use and report on how well it works +- [ ] Needs UX work for exposing non-recursive "HTTP transport" (NoFetch) over both libp2p and plain TCP (and sharing the configuration) +- [ ] Needs a mechanism for HTTP handler to signal supported features ([IPIP-425](https://github.com/ipfs/specs/pull/425)) +- [ ] Needs an option for Kubo to detect peers that have it enabled and prefer HTTP transport before falling back to bitswap (and use CAR if peer supports dag-scope=entity from [IPIP-402](https://github.com/ipfs/specs/pull/402)) From 3fa7ef8879a35b3f0474decb5b77d1d90437fcf7 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 6 Sep 2023 03:20:12 +0200 Subject: [PATCH 15/15] fix: run http+libp2p test against correct port seems we were testing regular gateway instead of proxied one --- .github/workflows/gateway-conformance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 06e2701d24d..d9131958502 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -197,14 +197,14 @@ jobs: run: | gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="") ./ipfs --api=/ip4/127.0.0.1/tcp/5002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1) - ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8082 /p2p/$gatewayNodeId + ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8092 /p2p/$gatewayNodeId working-directory: kubo-gateway/cmd/ipfs # 9. Run the gateway-conformance tests over libp2p - name: Run gateway-conformance tests over libp2p uses: ipfs/gateway-conformance/.github/actions/test@v0.3 with: - gateway-url: http://127.0.0.1:8081 + gateway-url: http://127.0.0.1:8092 json: output.json xml: output.xml html: output.html