feat(gateway)!: trustless mode

.github/workflows/gateway-sharness.yml

@@ -28,7 +28,6 @@ jobs:
         with:
           repository: ipfs/kubo
           path: kubo
-          ref: 503edee648e29c62888f05fa146ab13d9c65077d
       - name: Install Missing Tools
         run: sudo apt install -y socat net-tools fish libxml2-utils
       - name: Restore Go Cache

examples/gateway/common/handler.go

@@ -9,34 +9,43 @@ import (
 )
 
 func NewHandler(gwAPI gateway.IPFSBackend) http.Handler {
-	// Initialize the headers and gateway configuration. For this example, we do
-	// not add any special headers, but the required ones.
-	headers := map[string][]string{}
-	gateway.AddAccessControlHeaders(headers)
 	conf := gateway.Config{
-		Headers: headers,
-	}
+		// Initialize the headers. For this example, we do not add any special headers,
+		// only the required ones via gateway.AddAccessControlHeaders.
+		Headers: map[string][]string{},
 
-	// Initialize the public gateways that we will want to have available through
-	// Host header rewriting. This step is optional and only required if you're
-	// running multiple public gateways and want different settings and support
-	// for DNSLink and Subdomain Gateways.
-	noDNSLink := false // If you set DNSLink to point at the CID from CAR, you can load it!
-	publicGateways := map[string]*gateway.Specification{
-		// Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net
-		"example.net": {
-			Paths:         []string{"/ipfs", "/ipns"},
-			NoDNSLink:     noDNSLink,
-			UseSubdomains: true,
-		},
-		// Support local requests
-		"localhost": {
-			Paths:         []string{"/ipfs", "/ipns"},
-			NoDNSLink:     noDNSLink,
-			UseSubdomains: true,
+		// If you set DNSLink to point at the CID from CAR, you can load it!
+		NoDNSLink: false,
+
+		TrustedMode: true,
+
+		// Initialize the public gateways that we will want to have available through
+		// Host header rewriting. This step is optional and only required if you're
+		// running multiple public gateways and want different settings and support
+		// for DNSLink and Subdomain Gateways.
+		PublicGateways: map[string]*gateway.Specification{
+			// Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net
+			"example.net": {
+				Paths:         []string{"/ipfs", "/ipns"},
+				NoDNSLink:     false,
+				UseSubdomains: true,
+				// This gateway is used for testing and therefore we make non-trustless
+				// requests. Thus, we have to manually turn on the trusted mode.
+				TrustedMode: true,
+			},
+			// Support local requests
+			"localhost": {
+				Paths:         []string{"/ipfs", "/ipns"},
+				NoDNSLink:     false,
+				UseSubdomains: true,
+				TrustedMode:   true,
+			},
 		},
 	}
 
+	// Add required access control headers to the configuration.
+	gateway.AddAccessControlHeaders(conf.Headers)
+
 	// Creates a mux to serve the gateway paths. This is not strictly necessary
 	// and gwHandler could be used directly. However, on the next step we also want
 	// to add prometheus metrics, hence needing the mux.
@@ -56,7 +65,7 @@ func NewHandler(gwAPI gateway.IPFSBackend) http.Handler {
 	// or example.net. If you want to expose the metrics on such gateways,
 	// you will have to add the path "/debug" to the variable Paths.
 	var handler http.Handler
-	handler = gateway.WithHostname(mux, gwAPI, publicGateways, noDNSLink)
+	handler = gateway.WithHostname(conf, gwAPI, mux)
 
 	// Finally, wrap with the withConnect middleware. This is required since we use
 	// http.ServeMux which does not support CONNECT by default.

gateway/gateway.go

@@ -15,7 +15,61 @@ import (
 
 // Config is the configuration used when creating a new gateway handler.
 type Config struct {
+	// Headers is a map containing all the headers that should be sent by default
+	// in all requests. You can define custom headers, as well as add the recommended
+	// headers via AddAccessControlHeaders.
 	Headers map[string][]string
+
+	// TrustedMode configures this gateway to include trusted responses. By default,
+	// the gateway will only reply with trustless responses, as defined in the specs:
+	// https://specs.ipfs.tech/http-gateways/trustless-gateway/. This does NOT apply
+	// to localhost / 127.0.0.1 / ::1.
+	TrustedMode bool
+
+	// NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in
+	// response to requests with values in `Host` HTTP header. This flag can be
+	// overridden per FQDN in PublicGateways. To be used with WithHostname.
+	NoDNSLink bool
+
+	// PublicGateways configures the behavior of known public gateways. Each key is
+	// a fully qualified domain name (FQDN). To be used with WithHostname.
+	PublicGateways map[string]*Specification
+}
+
+// Specification is the specification of an IPFS Public Gateway.
+type Specification struct {
+	// Paths is explicit list of path prefixes that should be handled by
+	// this gateway. Example: `["/ipfs", "/ipns"]`
+	// Useful if you only want to support immutable `/ipfs`.
+	Paths []string
+
+	// UseSubdomains indicates whether or not this gateway uses subdomains
+	// for IPFS resources instead of paths. That is: http://CID.ipfs.GATEWAY/...
+	//
+	// If this flag is set, any /ipns/$id and/or /ipfs/$id paths in Paths
+	// will be permanently redirected to http://$id.[ipns|ipfs].$gateway/.
+	//
+	// We do not support using both paths and subdomains for a single domain
+	// for security reasons (Origin isolation).
+	UseSubdomains bool
+
+	// NoDNSLink configures this gateway to _not_ resolve DNSLink for the
+	// specific FQDN provided in `Host` HTTP header. Useful when you want to
+	// explicitly allow or refuse hosting a single hostname. To refuse all
+	// DNSLinks in `Host` processing, set NoDNSLink in Config instead. This setting
+	// overrides the global setting.
+	NoDNSLink bool
+
+	// InlineDNSLink configures this gateway to always inline DNSLink names
+	// (FQDN) into a single DNS label in order to interop with wildcard TLS certs
+	// and Origin per CID isolation provided by rules like https://publicsuffix.org
+	// This should be set to true if you use HTTPS.
+	InlineDNSLink bool
+
+	// TrustedMode configures this gateway to include trusted responses. By default,
+	// the gateway will only reply with trustless responses. This setting overrides
+	// the global setting.
+	TrustedMode bool
 }
 
 // TODO: Is this what we want for ImmutablePath?
@@ -221,7 +275,17 @@ func AddAccessControlHeaders(headers map[string][]string) {
 type RequestContextKey string
 
 const (
-	DNSLinkHostnameKey RequestContextKey = "dnslink-hostname"
+	// GatewayHostnameKey is the key for the hostname at which the gateway is
+	// operating. It may be a DNSLink, Subdomain or Regular gateway.
 	GatewayHostnameKey RequestContextKey = "gw-hostname"
-	ContentPathKey     RequestContextKey = "content-path"
+
+	// DNSLinkHostnameKey is the key for the hostname of a DNSLink Gateway:
+	// https://specs.ipfs.tech/http-gateways/dnslink-gateway/
+	DNSLinkHostnameKey RequestContextKey = "dnslink-hostname"
+
+	// SubdomainHostnameKey is the key for the hostname of a Subdomain Gateway:
+	// https://specs.ipfs.tech/http-gateways/subdomain-gateway/
+	SubdomainHostnameKey RequestContextKey = "subdomain-hostname"
+
+	ContentPathKey RequestContextKey = "content-path"
 )

gateway/gateway_test.go

@@ -198,14 +198,20 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, *mock
 }
 
 func newTestServer(t *testing.T, api IPFSBackend) *httptest.Server {
-	config := Config{Headers: map[string][]string{}}
+	return newTestServerWithConfig(t, api, Config{
+		Headers:     map[string][]string{},
+		TrustedMode: true,
+	})
+}
+
+func newTestServerWithConfig(t *testing.T, api IPFSBackend, config Config) *httptest.Server {
 	AddAccessControlHeaders(config.Headers)
 
 	handler := NewHandler(config, api)
 	mux := http.NewServeMux()
 	mux.Handle("/ipfs/", handler)
 	mux.Handle("/ipns/", handler)
-	handler = WithHostname(mux, api, map[string]*Specification{}, false)
+	handler = WithHostname(config, api, mux)
 
 	ts := httptest.NewServer(handler)
 	t.Cleanup(func() { ts.Close() })
@@ -544,3 +550,76 @@ func TestGoGetSupport(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusOK, res.StatusCode)
 }
+
+func TestTrustlessMode(t *testing.T) {
+	api, root := newMockAPI(t)
+	ts := newTestServerWithConfig(t, api, Config{
+		Headers:   map[string][]string{},
+		NoDNSLink: false,
+		PublicGateways: map[string]*Specification{
+			"trustless.com": {
+				Paths:       []string{"/ipfs", "/ipns"},
+				TrustedMode: false,
+			},
+			"trusted.com": {
+				Paths:       []string{"/ipfs", "/ipns"},
+				TrustedMode: true,
+			},
+		},
+	})
+	t.Logf("test server url: %s", ts.URL)
+
+	trustedFormats := []string{"", "dag-json", "dag-cbor", "tar", "json", "cbor"}
+	trustlessFormats := []string{"raw", "car"}
+
+	doRequests := func(formats []string, host string, expectedStatus int) {
+		for _, format := range formats {
+			req, err := http.NewRequest(http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/?format="+format, nil)
+			assert.Nil(t, err)
+
+			if host != "" {
+				req.Host = host
+			}
+
+			res, err := doWithoutRedirect(req)
+			assert.Nil(t, err)
+			defer res.Body.Close()
+			assert.Equal(t, expectedStatus, res.StatusCode)
+		}
+	}
+
+	t.Run("Explicit Trustless Gateway", func(t *testing.T) {
+		t.Parallel()
+		doRequests(trustlessFormats, "trustless.com", http.StatusOK)
+		doRequests(trustedFormats, "trustless.com", http.StatusNotImplemented)
+	})
+
+	t.Run("Explicit Trusted Gateway", func(t *testing.T) {
+		t.Parallel()
+		doRequests(trustlessFormats, "trusted.com", http.StatusOK)
+		doRequests(trustedFormats, "trusted.com", http.StatusOK)
+	})
+
+	t.Run("Implicit Default Trustless Gateway", func(t *testing.T) {
+		t.Parallel()
+		doRequests(trustlessFormats, "not.configured.com", http.StatusOK)
+		doRequests(trustedFormats, "not.configured.com", http.StatusNotImplemented)
+	})
+
+	t.Run("Implicit Default Local Trusted Gateway", func(t *testing.T) {
+		t.Parallel()
+		doRequests(trustlessFormats, "localhost", http.StatusOK)
+		doRequests(trustedFormats, "localhost", http.StatusOK)
+		doRequests(trustlessFormats, "127.0.0.1", http.StatusOK)
+		doRequests(trustedFormats, "127.0.0.1", http.StatusOK)
+		doRequests(trustlessFormats, "::1", http.StatusOK)
+		doRequests(trustedFormats, "::1", http.StatusOK)
+
+		doRequests(trustlessFormats, "localhost:8080", http.StatusOK)
+		doRequests(trustedFormats, "localhost:8080", http.StatusOK)
+		doRequests(trustlessFormats, "127.0.0.1:8080", http.StatusOK)
+		doRequests(trustedFormats, "127.0.0.1:8080", http.StatusOK)
+		doRequests(trustlessFormats, "[::1]:8080", http.StatusOK)
+		doRequests(trustedFormats, "[::1]:8080", http.StatusOK)
+	})
+}

gateway/handler.go

@@ -277,6 +277,19 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if !i.isTrustedMode(r) {
+		switch responseFormat {
+		case "application/vnd.ipld.raw",
+			"application/vnd.ipld.car",
+			"application/vnd.ipfs.ipns-record":
+			// Allowed
+		default:
+			err := errors.New("only trustless requests are accepted: https://specs.ipfs.tech/http-gateways/trustless-gateway/")
+			webError(w, err, http.StatusNotImplemented)
+			return
+		}
+	}
+
 	var success bool
 
 	// Support custom response formats passed via ?format or Accept HTTP header
@@ -314,6 +327,32 @@ func (i *handler) addUserHeaders(w http.ResponseWriter) {
 	}
 }
 
+func (i *handler) isTrustedMode(r *http.Request) bool {
+	// Get the host, by default the request's Host. If this request went through
+	// WithHostname, also check for the key in the context. If that is not present,
+	// also check X-Forwarded-Host to support reverse proxies.
+	host := r.Host
+	if h, ok := r.Context().Value(GatewayHostnameKey).(string); ok {
+		host = h
+	} else if xHost := r.Header.Get("X-Forwarded-Host"); xHost != "" {
+		host = xHost
+	}
+
+	// If the gateway is defined, return whatever is set.
+	if gw, ok := i.config.PublicGateways[host]; ok {
+		return gw.TrustedMode
+	}
+
+	// Cleanup the host and check if it's a local gateway. If so, it's trusted.
+	host = stripPort(host)
+	if host == "127.0.0.1" || host == "::1" || host == "localhost" {
+		return true
+	}
+
+	// Otherwise, the default.
+	return i.config.TrustedMode
+}
+
 func panicHandler(w http.ResponseWriter) {
 	if r := recover(); r != nil {
 		log.Error("A panic occurred in the gateway handler!")

gateway/handler_unixfs__redirects.go

@@ -210,10 +210,10 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat
 }
 
 func hasOriginIsolation(r *http.Request) bool {
-	_, gw := r.Context().Value(GatewayHostnameKey).(string)
+	_, subdomainGw := r.Context().Value(SubdomainHostnameKey).(string)
 	_, dnslink := r.Context().Value(DNSLinkHostnameKey).(string)
 
-	if gw || dnslink {
+	if subdomainGw || dnslink {
 		return true
 	}
 

gateway/handler_unixfs_dir.go

@@ -186,8 +186,10 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *
 	// for this request.
 	var gwURL string
 
-	// Get gateway hostname and build gateway URL.
-	if h, ok := r.Context().Value(GatewayHostnameKey).(string); ok {
+	// Ensure correct URL in DNSLink and Subdomain Gateways.
+	if h, ok := r.Context().Value(SubdomainHostnameKey).(string); ok {
+		gwURL = "//" + h
+	} else if h, ok := r.Context().Value(DNSLinkHostnameKey).(string); ok {
 		gwURL = "//" + h
 	} else {
 		gwURL = ""