feat: trustless-only mode (RAINBOW_TRUSTLESS_GATEWAY_DOMAINS) (#81)

* feat: trustless gateway option
* docs: RAINBOW_TRUSTLESS_GATEWAY_DOMAINS
* Apply suggestions from code review
* test: add trustless e2e test
* feat: print gateway domains when set

---------

Co-authored-by: Marcin Rataj <lidel@lidel.org>
Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com>

docs/environment-variables.md

@@ -5,6 +5,7 @@
 - [Configuration](#configuration)
   - [`RAINBOW_GATEWAY_DOMAINS`](#rainbow_gateway_domains)
   - [`RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS`](#rainbow_subdomain_gateway_domains)
+  - [`RAINBOW_TRUSTLESS_GATEWAY_DOMAINS`](#rainbow_trustless_gateway_domains)
   - [`KUBO_RPC_URL`](#kubo_rpc_url)
 - [Logging](#logging)
   - [`GOLOG_LOG_LEVEL`](#golog_log_level)
@@ -19,26 +20,56 @@
 
 ### `RAINBOW_GATEWAY_DOMAINS`
 
-Comma-separated list of path gateway hostnames. For example, passing `ipfs.io` will enable handler for standard [path gateway](https://specs.ipfs.tech/http-gateways/path-gateway/) requests with the `Host` header set to `ipfs.io`.
+Comma-separated list of [path gateway](https://specs.ipfs.tech/http-gateways/path-gateway/)
+hostnames that will serve both trustless and deserialized response types.
 
-Default: `127.0.0.1`
+Example:  passing `ipfs.io` will enable deserialized handler for flat
+[path gateway](https://specs.ipfs.tech/http-gateways/path-gateway/)
+requests with the `Host` header set to `ipfs.io`.
 
+Default: `127.0.0.1`
 
 ### `RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS`
 
-Comma-separated list of [subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) domains. For example, passing `dweb.link` will enable handler for standard [subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) requests with the `Host` header set to `*.ipfs.dweb.link` and  `*.ipns.dweb.link`.
+Comma-separated list of [subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/)
+domains for website hosting with Origin-isolation per content root.
+
+Example: passing `dweb.link` will enable handler for Origin-isolated
+[subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/)
+requests with the `Host` header with subdomain values matching
+`*.ipfs.dweb.link` or  `*.ipns.dweb.link`.
 
 Default: `localhost`
 
-### `KUBO_RPC_URL`
+### `RAINBOW_TRUSTLESS_GATEWAY_DOMAINS`
 
-Default: `127.0.0.1:5001` (see `DefaultKuboRPC`)
+Specifies trustless-only hostnames.
+
+Comma-separated list of [trustless gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
+domains, where unverified website asset hosting and deserialized responses is
+disabled, and **response types requested via `?format=` and `Accept` HTTP header are limited to
+[verifiable content types](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval)**:
+- [`application/vnd.ipld.raw`](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw)
+- [`application/vnd.ipld.car`](https://www.iana.org/assignments/media-types/application/vnd.ipld.car)
+- [`application/vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record)
+
+**NOTE:** This setting is applied on top of everything else, to ensure
+trustless domains can't be used for phishing or direct hotlinking and hosting of third-party content. Hostnames that are passed to both `RAINBOW_GATEWAY_DOMAINS` and `RAINBOW_TRUSTLESS_GATEWAY_DOMAINS` will work only as trustless gateways. 
+
+Example:  passing `trustless-gateway.link` will ensure only verifiable content types are supported
+when request comes with the `Host` header set to `trustless-gateway.link`.
+
+Default: none (`Host` is ignored and gateway at `127.0.0.1` supports both deserialized and verifiable response types)
+
+### `KUBO_RPC_URL`
 
 Single URL or a comma separated list of RPC endpoints that provide legacy `/api/v0` from Kubo.
 
 We use this to redirect some legacy `/api/v0` commands that need to be handled on `ipfs.io`.
 
-This is deprecated and will be removed in the future.
+**NOTE:** This is deprecated and will be removed in the future.
+
+Default: `127.0.0.1:5001` (see `DefaultKuboRPC`)
 
 ## Logging
 

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/mitchellh/go-server-timing v1.0.1
 	github.com/mr-tron/base58 v1.2.0
 	github.com/multiformats/go-multiaddr v0.12.2
+	github.com/multiformats/go-multicodec v0.9.0
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/prometheus/client_golang v1.18.0
 	github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
@@ -126,7 +127,6 @@ require (
 	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.2.0 // indirect
-	github.com/multiformats/go-multicodec v0.9.0 // indirect
 	github.com/multiformats/go-multihash v0.2.3 // indirect
 	github.com/multiformats/go-multistream v0.5.0 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
@@ -153,6 +153,7 @@ require (
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20240109153615-66e95c3e8a87 // indirect
+	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.opentelemetry.io/contrib/propagators/aws v1.23.0 // indirect

handler_test.go

@@ -1,11 +1,22 @@
 package main
 
 import (
+	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
+	chunker "github.com/ipfs/boxo/chunker"
+	"github.com/ipfs/boxo/ipld/merkledag"
+	"github.com/ipfs/boxo/ipld/unixfs/importer/balanced"
+	uih "github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
+	util "github.com/ipfs/boxo/util"
+	"github.com/ipfs/go-cid"
+	ic "github.com/libp2p/go-libp2p/core/crypto"
+	"github.com/multiformats/go-multicodec"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type rpcRedirectTest struct {
@@ -85,3 +96,101 @@ func TestRPCNotImplemented(t *testing.T) {
 		assert.Equal(t, http.StatusNotImplemented, resp.Code)
 	}
 }
+
+func mustTestServer(t *testing.T, cfg Config) (*httptest.Server, *Node) {
+	cfg.DataDir = t.TempDir()
+	cfg.BlockstoreType = "flatfs"
+
+	ctx := context.Background()
+
+	sr := util.NewTimeSeededRand()
+	sk, _, err := ic.GenerateKeyPairWithReader(ic.Ed25519, 2048, sr)
+	require.NoError(t, err)
+
+	cdns := newCachedDNS(dnsCacheRefreshInterval)
+
+	t.Cleanup(func() {
+		_ = cdns.Close()
+	})
+
+	gnd, err := Setup(ctx, cfg, sk, cdns)
+	if err != nil {
+		require.NoError(t, err)
+	}
+
+	handler, err := setupGatewayHandler(cfg, gnd)
+	if err != nil {
+		require.NoError(t, err)
+	}
+
+	ts := httptest.NewServer(handler)
+
+	return ts, gnd
+}
+
+func mustAddFile(t *testing.T, gnd *Node, content []byte) cid.Cid {
+	dsrv := merkledag.NewDAGService(gnd.bsrv)
+
+	// Create a UnixFS graph from our file, parameters described here but can be visualized at https://dag.ipfs.tech/
+	ufsImportParams := uih.DagBuilderParams{
+		Maxlinks:  uih.DefaultLinksPerBlock, // Default max of 174 links per block
+		RawLeaves: true,                     // Leave the actual file bytes untouched instead of wrapping them in a dag-pb protobuf wrapper
+		CidBuilder: cid.V1Builder{ // Use CIDv1 for all links
+			Codec:    uint64(multicodec.DagPb),
+			MhType:   uint64(multicodec.Sha2_256), // Use SHA2-256 as the hash function
+			MhLength: -1,                          // Use the default hash length for the given hash function (in this case 256 bits)
+		},
+		Dagserv: dsrv,
+		NoCopy:  false,
+	}
+	ufsBuilder, err := ufsImportParams.New(chunker.NewSizeSplitter(bytes.NewReader(content), chunker.DefaultBlockSize)) // Split the file up into fixed sized 256KiB chunks
+	require.NoError(t, err)
+
+	nd, err := balanced.Layout(ufsBuilder) // Arrange the graph with a balanced layout
+	require.NoError(t, err)
+
+	return nd.Cid()
+}
+
+func TestTrustless(t *testing.T) {
+	t.Parallel()
+
+	ts, gnd := mustTestServer(t, Config{
+		TrustlessGatewayDomains: []string{"trustless.com"},
+	})
+
+	content := "hello world"
+	cid := mustAddFile(t, gnd, []byte(content))
+	url := ts.URL + "/ipfs/" + cid.String()
+
+	t.Run("Non-trustless request returns 406", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		require.NoError(t, err)
+		req.Host = "trustless.com"
+
+		res, err := http.DefaultClient.Do(req)
+		assert.NoError(t, err)
+		assert.Equal(t, http.StatusNotAcceptable, res.StatusCode)
+	})
+
+	t.Run("Trustless request with query parameter returns 200", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, url+"?format=raw", nil)
+		require.NoError(t, err)
+		req.Host = "trustless.com"
+
+		res, err := http.DefaultClient.Do(req)
+		assert.NoError(t, err)
+		assert.Equal(t, http.StatusOK, res.StatusCode)
+	})
+
+	t.Run("Trustless request with accept header returns 200", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		require.NoError(t, err)
+		req.Host = "trustless.com"
+		req.Header.Set("Accept", "application/vnd.ipld.raw")
+
+		res, err := http.DefaultClient.Do(req)
+		assert.NoError(t, err)
+		assert.Equal(t, http.StatusOK, res.StatusCode)
+	})
+}

handlers.go

@@ -135,6 +135,16 @@ func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) {
 		}
 	}
 
+	for _, domain := range cfg.TrustlessGatewayDomains {
+		publicGateways[domain] = &gateway.PublicGateway{
+			Paths:                 []string{"/ipfs", "/ipns", "/version"},
+			NoDNSLink:             true,
+			InlineDNSLink:         true,
+			DeserializedResponses: false,
+			UseSubdomains:         contains(cfg.SubdomainGatewayDomains, domain),
+		}
+	}
+
 	// If we're doing tests, ensure the right public gateways are enabled.
 	if os.Getenv("GATEWAY_CONFORMANCE_TEST") == "true" {
 		publicGateways["example.com"] = &gateway.PublicGateway{
@@ -340,3 +350,13 @@ func BlockProfileRateOption(path string, mux *http.ServeMux) *http.ServeMux {
 	})
 	return mux
 }
+
+func contains[T comparable](collection []T, element T) bool {
+	for _, item := range collection {
+		if item == element {
+			return true
+		}
+	}
+
+	return false
+}

main.go

@@ -46,12 +46,12 @@ only websites, but any supported content-addressed Merkle-DAG), in formats
 that are suitable for verification client-side (i.e. CAR files).
 
 Rainbow is optimized to perform the tasks of a gateway and only that, making
-opinionated choices on the configration and setup of internal
+opinionated choices on the configuration and setup of internal
 components. Rainbow aims to serve production environments, where gateways are
 deployed as a public service meant to be accessible by anyone. Rainbow acts as
 a client to the IPFS network and does not serve or provide content to
 it. Rainbow cannot be used to store or pin IPFS content, other than that
-temporailly served over HTTP. Rainbow is just a gateway.
+temporarily served over HTTP. Rainbow is just a gateway.
 
 Persistent configuration and data is stored in $RAINBOW_DATADIR (by default,
 the folder in which rainbow is run).
@@ -73,7 +73,6 @@ Generate an identity seed and launch a gateway:
 `
 
 	app.Flags = []cli.Flag{
-
 		&cli.StringFlag{
 			Name:    "datadir",
 			Value:   "",
@@ -96,13 +95,19 @@ Generate an identity seed and launch a gateway:
 			Name:    "gateway-domains",
 			Value:   "",
 			EnvVars: []string{"RAINBOW_GATEWAY_DOMAINS"},
-			Usage:   "Legacy path-gateway domains. Comma-separated list.",
+			Usage:   "Domains with flat path gateway, no Origin isolation. Comma-separated list.",
 		},
 		&cli.StringFlag{
 			Name:    "subdomain-gateway-domains",
 			Value:   "",
 			EnvVars: []string{"RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS"},
-			Usage:   "Subdomain gateway domains. Comma-separated list.",
+			Usage:   "Domains with subdomain-based Origin isolation. Comma-separated list.",
+		},
+		&cli.StringFlag{
+			Name:    "trustless-gateway-domains",
+			Value:   "",
+			EnvVars: []string{"RAINBOW_TRUSTLESS_GATEWAY_DOMAINS"},
+			Usage:   "Domains limited to trustless, verifiable response types. Comma-separated list.",
 		},
 		&cli.StringFlag{
 			Name:    "gateway-listen-address",
@@ -116,7 +121,6 @@ Generate an identity seed and launch a gateway:
 			EnvVars: []string{"RAINBOW_CTL_LISTEN_ADDRESS"},
 			Usage:   "Listen address for the management api and metrics",
 		},
-
 		&cli.IntFlag{
 			Name:    "connmgr-low",
 			Value:   100,
@@ -270,6 +274,7 @@ share the same seed as long as the indexes are different.
 			BlockstoreType:          cctx.String("blockstore"),
 			GatewayDomains:          getCommaSeparatedList(cctx.String("gateway-domains")),
 			SubdomainGatewayDomains: getCommaSeparatedList(cctx.String("subdomain-gateway-domains")),
+			TrustlessGatewayDomains: getCommaSeparatedList(cctx.String("trustless-gateway-domains")),
 			ConnMgrLow:              cctx.Int("connmgr-low"),
 			ConnMgrHi:               cctx.Int("connmgr-high"),
 			ConnMgrGrace:            cctx.Duration("connmgr-grace"),
@@ -334,10 +339,16 @@ share the same seed as long as the indexes are different.
 		var wg sync.WaitGroup
 		wg.Add(2)
 
-		fmt.Printf("Gateway listening at %s\n", gatewayListen)
-		fmt.Printf("Legacy RPC at /api/v0 (%s): %s\n", EnvKuboRPC, strings.Join(gnd.kuboRPCs, " "))
+		fmt.Printf("IPFS Gateway listening at %s\n\n", gatewayListen)
+
+		printIfListConfigured("  RAINBOW_GATEWAY_DOMAINS           = ", cfg.GatewayDomains)
+		printIfListConfigured("  RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS = ", cfg.SubdomainGatewayDomains)
+		printIfListConfigured("  RAINBOW_TRUSTLESS_GATEWAY_DOMAINS = ", cfg.TrustlessGatewayDomains)
+		printIfListConfigured("  Legacy RPC at /api/v0 will redirect to KUBO_RPC_URL = ", cfg.KuboRPCURLs)
+
+		fmt.Printf("\n")
 		fmt.Printf("CTL endpoint listening at http://%s\n", ctlListen)
-		fmt.Printf("Metrics: http://%s/debug/metrics/prometheus\n\n", ctlListen)
+		fmt.Printf("  Metrics: http://%s/debug/metrics/prometheus\n\n", ctlListen)
 
 		go func() {
 			defer wg.Done()
@@ -424,3 +435,9 @@ func getCommaSeparatedList(val string) []string {
 	}
 	return items
 }
+
+func printIfListConfigured(message string, list []string) {
+	if len(list) > 0 {
+		fmt.Printf(message+"%v\n", strings.Join(list, ", "))
+	}
+}

setup.go

@@ -96,6 +96,7 @@ type Config struct {
 
 	GatewayDomains          []string
 	SubdomainGatewayDomains []string
+	TrustlessGatewayDomains []string
 	RoutingV1               string
 	KuboRPCURLs             []string
 	DHTSharedHost           bool