Merge branch 'master' into changelog-quic

.github/workflows/gateway-conformance.yml

@@ -17,7 +17,24 @@ defaults:
   run:
     shell: bash
 
+env:
+  # hostnames expected by https://github.com/ipfs/gateway-conformance
+  GATEWAY_PUBLIC_GATEWAYS: |
+    {
+      "example.com": {
+        "UseSubdomains": true,
+        "InlineDNSLink": true,
+        "Paths": ["/ipfs", "/ipns"]
+      },
+      "localhost": {
+        "UseSubdomains": true,
+        "InlineDNSLink": true,
+        "Paths": ["/ipfs", "/ipns"]
+      }
+    }
+
 jobs:
+  # Testing all gateway features via TCP port specified in Addresses.Gateway
   gateway-conformance:
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -33,6 +50,9 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: 1.21.x
+      - uses: protocol/cache-go-action@v1
+        with:
+          name: ${{ github.job }}
       - name: Checkout kubo-gateway
         uses: actions/checkout@v4
         with:
@@ -43,22 +63,8 @@ jobs:
 
       # 3. Init the kubo-gateway
       - name: Init kubo-gateway
-        env:
-          GATEWAY_PUBLIC_GATEWAYS: |
-            {
-              "example.com": {
-                "UseSubdomains": true,
-                "InlineDNSLink": true,
-                "Paths": ["/ipfs", "/ipns"]
-              },
-              "localhost": {
-                "UseSubdomains": true,
-                "InlineDNSLink": true,
-                "Paths": ["/ipfs", "/ipns"]
-              }
-            }
         run: |
-          ./ipfs init
+          ./ipfs init -e
           ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
         working-directory: kubo-gateway/cmd/ipfs
 
@@ -115,3 +121,109 @@ jobs:
         with:
           name: gateway-conformance.json
           path: output.json
+
+  # Testing trustless gateway feature subset exposed as libp2p protocol
+  gateway-conformance-libp2p-experiment:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3
+        with:
+          output: fixtures
+
+      # 2. Build the kubo-gateway
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.20.x
+      - uses: protocol/cache-go-action@v1
+        with:
+          name: ${{ github.job }}
+      - name: Checkout kubo-gateway
+        uses: actions/checkout@v3
+        with:
+          path: kubo-gateway
+      - name: Build kubo-gateway
+        run: make build
+        working-directory: kubo-gateway
+
+      # 3. Init the kubo-gateway
+      - name: Init kubo-gateway
+        run: |
+          ./ipfs init --profile=test
+          ./ipfs config --json Gateway.PublicGateways "$GATEWAY_PUBLIC_GATEWAYS"
+          ./ipfs config --json Experimental.GatewayOverLibp2p true
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 4. Populate the Kubo gateway with the gateway-conformance fixtures
+      - name: Import fixtures
+        run: |
+          # Import car files
+          find ./fixtures -name '*.car' -exec kubo-gateway/cmd/ipfs/ipfs dag import --pin-roots=false {} \;
+
+      # 5. Start the kubo-gateway
+      - name: Start kubo-gateway
+        run: |
+          ( ./ipfs daemon & ) | sed '/Daemon is ready/q'
+          while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 6. Setup a kubo http-p2p-proxy to expose libp2p protocol as a regular HTTP port for gateway conformance tests
+      - name: Init p2p-proxy kubo node
+        env:
+          IPFS_PATH: "~/.kubo-p2p-proxy"
+        run: |
+          ./ipfs init --profile=test -e
+          ./ipfs config --json Experimental.Libp2pStreamMounting true
+          ./ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8081"
+          ./ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5002"
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 7. Start the kubo http-p2p-proxy
+      - name: Start kubo http-p2p-proxy
+        env:
+          IPFS_PATH: "~/.kubo-p2p-proxy"
+        run: |
+          ( ./ipfs daemon & ) | sed '/Daemon is ready/q'
+          while [[ "$(./ipfs id | jq '.Addresses | length')" == '0' ]]; do sleep 1; done
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 8. Start forwarding data from the http-p2p-proxy to the node serving the Gateway API over libp2p
+      - name: Start http-over-libp2p forwarding proxy
+        run: |
+          gatewayNodeId=$(./ipfs --api=/ip4/127.0.0.1/tcp/5001 id -f="<id>")
+          ./ipfs --api=/ip4/127.0.0.1/tcp/5002 swarm connect $(./ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1)
+          ./ipfs --api=/ip4/127.0.0.1/tcp/5002 p2p forward --allow-custom-protocol /http/1.1 /ip4/127.0.0.1/tcp/8092 /p2p/$gatewayNodeId
+        working-directory: kubo-gateway/cmd/ipfs
+
+      # 9. Run the gateway-conformance tests over libp2p
+      - name: Run gateway-conformance tests over libp2p
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.3
+        with:
+          gateway-url: http://127.0.0.1:8092
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          args: --specs "trustless-gateway,-trustless-ipns-gateway" -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+
+      # 10. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v3
+        with:
+          name: gateway-conformance-libp2p.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v3
+        with:
+          name: gateway-conformance-libp2p.json
+          path: output.json

cmd/ipfs/daemon.go

@@ -15,6 +15,9 @@ import (
 
 	multierror "github.com/hashicorp/go-multierror"
 
+	options "github.com/ipfs/boxo/coreiface/options"
+	cmds "github.com/ipfs/go-ipfs-cmds"
+	mprome "github.com/ipfs/go-metrics-prometheus"
 	version "github.com/ipfs/kubo"
 	utilmain "github.com/ipfs/kubo/cmd/ipfs/util"
 	oldcmds "github.com/ipfs/kubo/commands"
@@ -30,14 +33,12 @@ import (
 	fsrepo "github.com/ipfs/kubo/repo/fsrepo"
 	"github.com/ipfs/kubo/repo/fsrepo/migrations"
 	"github.com/ipfs/kubo/repo/fsrepo/migrations/ipfsfetcher"
+	goprocess "github.com/jbenet/goprocess"
 	p2pcrypto "github.com/libp2p/go-libp2p/core/crypto"
 	pnet "github.com/libp2p/go-libp2p/core/pnet"
+	"github.com/libp2p/go-libp2p/core/protocol"
+	p2phttp "github.com/libp2p/go-libp2p/p2p/http"
 	sockets "github.com/libp2p/go-socket-activation"
-
-	options "github.com/ipfs/boxo/coreiface/options"
-	cmds "github.com/ipfs/go-ipfs-cmds"
-	mprome "github.com/ipfs/go-metrics-prometheus"
-	goprocess "github.com/jbenet/goprocess"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
 	prometheus "github.com/prometheus/client_golang/prometheus"
@@ -551,6 +552,12 @@ take effect.
 		return err
 	}
 
+	// add trustless gateway over libp2p
+	p2pGwErrc, err := serveTrustlessGatewayOverLibp2p(cctx)
+	if err != nil {
+		return err
+	}
+
 	// Add ipfs version info to prometheus metrics
 	ipfsInfoMetric := promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "ipfs_info",
@@ -617,7 +624,7 @@ take effect.
 	// collect long-running errors and block for shutdown
 	// TODO(cryptix): our fuse currently doesn't follow this pattern for graceful shutdown
 	var errs error
-	for err := range merge(apiErrc, gwErrc, gcErrc) {
+	for err := range merge(apiErrc, gwErrc, gcErrc, p2pGwErrc) {
 		if err != nil {
 			errs = multierror.Append(errs, err)
 		}
@@ -899,6 +906,61 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 	return errc, nil
 }
 
+const gatewayProtocolID protocol.ID = "/ipfs/gateway" // FIXME: specify https://github.com/ipfs/specs/issues/433
+
+func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error) {
+	node, err := cctx.ConstructNode()
+	if err != nil {
+		return nil, fmt.Errorf("serveHTTPGatewayOverLibp2p: ConstructNode() failed: %s", err)
+	}
+	cfg, err := node.Repo.Config()
+	if err != nil {
+		return nil, fmt.Errorf("could not read config: %w", err)
+	}
+
+	if !cfg.Experimental.GatewayOverLibp2p {
+		errCh := make(chan error)
+		close(errCh)
+		return errCh, nil
+	}
+
+	opts := []corehttp.ServeOption{
+		corehttp.MetricsCollectionOption("libp2p-gateway"),
+		corehttp.Libp2pGatewayOption(),
+		corehttp.VersionOption(),
+	}
+
+	handler, err := corehttp.MakeHandler(node, nil, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	h := p2phttp.Host{
+		StreamHost: node.PeerHost,
+	}
+
+	tmpProtocol := protocol.ID("/kubo/delete-me")
+	h.SetHTTPHandler(tmpProtocol, http.NotFoundHandler())
+	h.WellKnownHandler.RemoveProtocolMeta(tmpProtocol)
+
+	h.WellKnownHandler.AddProtocolMeta(gatewayProtocolID, p2phttp.ProtocolMeta{Path: "/"})
+	h.ServeMux = http.NewServeMux()
+	h.ServeMux.Handle("/", handler)
+
+	errc := make(chan error, 1)
+	go func() {
+		defer close(errc)
+		errc <- h.Serve()
+	}()
+
+	go func() {
+		<-node.Process.Closing()
+		h.Close()
+	}()
+
+	return errc, nil
+}
+
 // collects options and opens the fuse mountpoint.
 func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error {
 	cfg, err := cctx.GetConfig()

config/experiments.go

@@ -11,4 +11,5 @@ type Experiments struct {
 	AcceleratedDHTClient          experimentalAcceleratedDHTClient `json:",omitempty"`
 	OptimisticProvide             bool
 	OptimisticProvideJobsPoolSize int
+	GatewayOverLibp2p             bool `json:",omitempty"`
 }

core/corehttp/commands.go

@@ -166,7 +166,7 @@ func CommandsROOption(cctx oldcmds.Context) ServeOption {
 func CheckVersionOption() ServeOption {
 	daemonVersion := version.ApiVersion
 
-	return ServeOption(func(n *core.IpfsNode, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) {
+	return func(n *core.IpfsNode, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) {
 		mux := http.NewServeMux()
 		parent.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 			if strings.HasPrefix(r.URL.Path, APIPath) {
@@ -188,5 +188,5 @@ func CheckVersionOption() ServeOption {
 		})
 
 		return mux, nil
-	})
+	}
 }

core/corehttp/corehttp.go

@@ -31,9 +31,9 @@ const shutdownTimeout = 30 * time.Second
 // initially passed in if not.
 type ServeOption func(*core.IpfsNode, net.Listener, *http.ServeMux) (*http.ServeMux, error)
 
-// makeHandler turns a list of ServeOptions into a http.Handler that implements
+// MakeHandler turns a list of ServeOptions into a http.Handler that implements
 // all of the given options, in order.
-func makeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http.Handler, error) {
+func MakeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http.Handler, error) {
 	topMux := http.NewServeMux()
 	mux := topMux
 	for _, option := range options {
@@ -86,7 +86,7 @@ func Serve(node *core.IpfsNode, lis net.Listener, options ...ServeOption) error
 	// make sure we close this no matter what.
 	defer lis.Close()
 
-	handler, err := makeHandler(node, lis, options...)
+	handler, err := MakeHandler(node, lis, options...)
 	if err != nil {
 		return err
 	}

core/corehttp/gateway.go

@@ -41,7 +41,7 @@ func GatewayOption(paths ...string) ServeOption {
 		handler = otelhttp.NewHandler(handler, "Gateway")
 
 		for _, p := range paths {
-			mux.HandleFunc(p+"/", handler.ServeHTTP)
+			mux.Handle(p+"/", handler)
 		}
 
 		return mux, nil
@@ -61,7 +61,7 @@ func HostnameOption() ServeOption {
 		}
 
 		childMux := http.NewServeMux()
-		mux.HandleFunc("/", gateway.NewHostnameHandler(config, backend, childMux).ServeHTTP)
+		mux.Handle("/", gateway.NewHostnameHandler(config, backend, childMux))
 		return childMux, nil
 	}
 }
@@ -76,6 +76,31 @@ func VersionOption() ServeOption {
 	}
 }
 
+func Libp2pGatewayOption() ServeOption {
+	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
+		bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
+
+		backend, err := gateway.NewBlocksBackend(bserv)
+		if err != nil {
+			return nil, err
+		}
+
+		gwConfig := gateway.Config{
+			DeserializedResponses: false,
+			NoDNSLink:             true,
+			PublicGateways:        nil,
+			Menu:                  nil,
+		}
+
+		handler := gateway.NewHandler(gwConfig, &offlineGatewayErrWrapper{gwimpl: backend})
+		handler = otelhttp.NewHandler(handler, "Libp2p-Gateway")
+
+		mux.Handle("/ipfs/", handler)
+
+		return mux, nil
+	}
+}
+
 func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
 	cfg, err := n.Repo.Config()
 	if err != nil {

core/corehttp/gateway_test.go

@@ -121,7 +121,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface
 	ts := httptest.NewServer(dh)
 	t.Cleanup(func() { ts.Close() })
 
-	dh.Handler, err = makeHandler(n,
+	dh.Handler, err = MakeHandler(n,
 		ts.Listener,
 		HostnameOption(),
 		GatewayOption("/ipfs", "/ipns"),