diff --git a/examples/gateway/common/handler.go b/examples/gateway/common/handler.go index f8782fbc8..648d454bb 100644 --- a/examples/gateway/common/handler.go +++ b/examples/gateway/common/handler.go @@ -10,34 +10,48 @@ import ( ) func NewHandler(gwAPI gateway.IPFSBackend) http.Handler { - // Initialize the headers and gateway configuration. For this example, we do - // not add any special headers, but the required ones. - headers := map[string][]string{} - gateway.AddAccessControlHeaders(headers) conf := gateway.Config{ - Headers: headers, - } + // Initialize the headers. For this example, we do not add any special headers, + // only the required ones via gateway.AddAccessControlHeaders. + Headers: map[string][]string{}, - // Initialize the public gateways that we will want to have available through - // Host header rewriting. This step is optional and only required if you're - // running multiple public gateways and want different settings and support - // for DNSLink and Subdomain Gateways. - noDNSLink := false // If you set DNSLink to point at the CID from CAR, you can load it! - publicGateways := map[string]*gateway.Specification{ - // Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net - "example.net": { - Paths: []string{"/ipfs", "/ipns"}, - NoDNSLink: noDNSLink, - UseSubdomains: true, - }, - // Support local requests - "localhost": { - Paths: []string{"/ipfs", "/ipns"}, - NoDNSLink: noDNSLink, - UseSubdomains: true, + // If you set DNSLink to point at the CID from CAR, you can load it! + NoDNSLink: false, + + // For these examples we have the trusted mode enabled by default. That is, + // all types of requests will be accepted. By default, only Trustless Gateway + // requests work: https://specs.ipfs.tech/http-gateways/trustless-gateway/ + DeserializedResponses: true, + + // Initialize the public gateways that we will want to have available + // through Host header rewriting. This step is optional, but required + // if you're running multiple public gateways on different hostnames + // and want different settings such as support for Deserialized + // Responses on localhost, or DNSLink and Subdomain Gateways. + PublicGateways: map[string]*gateway.Specification{ + // Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net + "example.net": { + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: false, + UseSubdomains: true, + // This subdomain gateway is used for testing and therefore we make non-trustless requests. + DeserializedResponses: true, + }, + // Support local requests + "localhost": { + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: false, + UseSubdomains: true, + // Localhost is considered trusted, ok to allow deserialized responses + // as long it is not exposed to the internet. + DeserializedResponses: true, + }, }, } + // Add required access control headers to the configuration. + gateway.AddAccessControlHeaders(conf.Headers) + // Creates a mux to serve the gateway paths. This is not strictly necessary // and gwHandler could be used directly. However, on the next step we also want // to add prometheus metrics, hence needing the mux. @@ -57,7 +71,7 @@ func NewHandler(gwAPI gateway.IPFSBackend) http.Handler { // or example.net. If you want to expose the metrics on such gateways, // you will have to add the path "/debug" to the variable Paths. var handler http.Handler - handler = gateway.WithHostname(mux, gwAPI, publicGateways, noDNSLink) + handler = gateway.WithHostname(conf, gwAPI, mux) // Then, wrap with the withConnect middleware. This is required since we use // http.ServeMux which does not support CONNECT by default. diff --git a/gateway/gateway.go b/gateway/gateway.go index b6f33da64..15e91422e 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -15,7 +15,69 @@ import ( // Config is the configuration used when creating a new gateway handler. type Config struct { + // Headers is a map containing all the headers that should be sent by default + // in all requests. You can define custom headers, as well as add the recommended + // headers via AddAccessControlHeaders. Headers map[string][]string + + // DeserializedResponses configures this gateway to support returning data + // in deserialized format. By default, the gateway will only support + // trustless, verifiable [application/vnd.ipld.raw] and + // [application/vnd.ipld.car] responses, operating as a [Trustless Gateway]. + // + // This global flag can be overridden per FQDN in PublicGateways map. + // + // [application/vnd.ipld.raw]: https://www.iana.org/assignments/media-types/application/vnd.ipld.raw + // [application/vnd.ipld.car]: https://www.iana.org/assignments/media-types/application/vnd.ipld.car + // [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/ + DeserializedResponses bool + + // NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in + // response to requests with values in `Host` HTTP header. This flag can be + // overridden per FQDN in PublicGateways. To be used with WithHostname. + NoDNSLink bool + + // PublicGateways configures the behavior of known public gateways. Each key is + // a fully qualified domain name (FQDN). To be used with WithHostname. + PublicGateways map[string]*Specification +} + +// Specification is the specification of an IPFS Public Gateway. +type Specification struct { + // Paths is explicit list of path prefixes that should be handled by + // this gateway. Example: `["/ipfs", "/ipns"]` + // Useful if you only want to support immutable `/ipfs`. + Paths []string + + // UseSubdomains indicates whether or not this is a [Subdomain Gateway]. + // + // If this flag is set, any `/ipns/$id` and/or `/ipfs/$id` paths in Paths + // will be permanently redirected to `http(s)://$id.[ipns|ipfs].$gateway/`. + // + // We do not support using both paths and subdomains for a single domain + // for security reasons ([Origin isolation]). + // + // [Subdomain Gateway]: https://specs.ipfs.tech/http-gateways/subdomain-gateway/ + // [Origin isolation]: https://en.wikipedia.org/wiki/Same-origin_policy + UseSubdomains bool + + // NoDNSLink configures this gateway to _not_ resolve DNSLink for the + // specific FQDN provided in `Host` HTTP header. Useful when you want to + // explicitly allow or refuse hosting a single hostname. To refuse all + // DNSLinks in `Host` processing, set NoDNSLink in Config instead. This setting + // overrides the global setting. + NoDNSLink bool + + // InlineDNSLink configures this gateway to always inline DNSLink names + // (FQDN) into a single DNS label in order to interop with wildcard TLS certs + // and Origin per CID isolation provided by rules like https://publicsuffix.org + // + // This should be set to true if you use HTTPS. + InlineDNSLink bool + + // DeserializedResponses configures this gateway to support returning data + // in deserialized format. This setting overrides the global setting. + DeserializedResponses bool } // TODO: Is this what we want for ImmutablePath? @@ -221,7 +283,17 @@ func AddAccessControlHeaders(headers map[string][]string) { type RequestContextKey string const ( - DNSLinkHostnameKey RequestContextKey = "dnslink-hostname" + // GatewayHostnameKey is the key for the hostname at which the gateway is + // operating. It may be a DNSLink, Subdomain or Regular gateway. GatewayHostnameKey RequestContextKey = "gw-hostname" - ContentPathKey RequestContextKey = "content-path" + + // DNSLinkHostnameKey is the key for the hostname of a DNSLink Gateway: + // https://specs.ipfs.tech/http-gateways/dnslink-gateway/ + DNSLinkHostnameKey RequestContextKey = "dnslink-hostname" + + // SubdomainHostnameKey is the key for the hostname of a Subdomain Gateway: + // https://specs.ipfs.tech/http-gateways/subdomain-gateway/ + SubdomainHostnameKey RequestContextKey = "subdomain-hostname" + + ContentPathKey RequestContextKey = "content-path" ) diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index ef14eeeba..2e8e2e28b 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -198,14 +198,20 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, *mock } func newTestServer(t *testing.T, api IPFSBackend) *httptest.Server { - config := Config{Headers: map[string][]string{}} + return newTestServerWithConfig(t, api, Config{ + Headers: map[string][]string{}, + DeserializedResponses: true, + }) +} + +func newTestServerWithConfig(t *testing.T, api IPFSBackend, config Config) *httptest.Server { AddAccessControlHeaders(config.Headers) handler := NewHandler(config, api) mux := http.NewServeMux() mux.Handle("/ipfs/", handler) mux.Handle("/ipns/", handler) - handler = WithHostname(mux, api, map[string]*Specification{}, false) + handler = WithHostname(config, api, mux) ts := httptest.NewServer(handler) t.Cleanup(func() { ts.Close() }) @@ -573,3 +579,128 @@ func TestIpnsBase58MultihashRedirect(t *testing.T) { assert.Equal(t, "/ipns/k2k4r8ol4m8kkcqz509c1rcjwunebj02gcnm5excpx842u736nja8ger?keep=query", res.Header.Get("Location")) }) } + +func TestIpfsTrustlessMode(t *testing.T) { + api, root := newMockAPI(t) + + ts := newTestServerWithConfig(t, api, Config{ + Headers: map[string][]string{}, + NoDNSLink: false, + PublicGateways: map[string]*Specification{ + "trustless.com": { + Paths: []string{"/ipfs", "/ipns"}, + }, + "trusted.com": { + Paths: []string{"/ipfs", "/ipns"}, + DeserializedResponses: true, + }, + }, + }) + t.Logf("test server url: %s", ts.URL) + + trustedFormats := []string{"", "dag-json", "dag-cbor", "tar", "json", "cbor"} + trustlessFormats := []string{"raw", "car"} + + doRequest := func(t *testing.T, path, host string, expectedStatus int) { + req, err := http.NewRequest(http.MethodGet, ts.URL+path, nil) + assert.Nil(t, err) + + if host != "" { + req.Host = host + } + + res, err := doWithoutRedirect(req) + assert.Nil(t, err) + defer res.Body.Close() + assert.Equal(t, expectedStatus, res.StatusCode) + } + + doIpfsCidRequests := func(t *testing.T, formats []string, host string, expectedStatus int) { + for _, format := range formats { + doRequest(t, "/ipfs/"+root.String()+"/?format="+format, host, expectedStatus) + } + } + + doIpfsCidPathRequests := func(t *testing.T, formats []string, host string, expectedStatus int) { + for _, format := range formats { + doRequest(t, "/ipfs/"+root.String()+"/EmptyDir/?format="+format, host, expectedStatus) + } + } + + trustedTests := func(t *testing.T, host string) { + doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK) + doIpfsCidRequests(t, trustedFormats, host, http.StatusOK) + doIpfsCidPathRequests(t, trustlessFormats, host, http.StatusOK) + doIpfsCidPathRequests(t, trustedFormats, host, http.StatusOK) + } + + trustlessTests := func(t *testing.T, host string) { + doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK) + doIpfsCidRequests(t, trustedFormats, host, http.StatusNotAcceptable) + doIpfsCidPathRequests(t, trustlessFormats, host, http.StatusNotAcceptable) + doIpfsCidPathRequests(t, trustedFormats, host, http.StatusNotAcceptable) + } + + t.Run("Explicit Trustless Gateway", func(t *testing.T) { + t.Parallel() + trustlessTests(t, "trustless.com") + }) + + t.Run("Explicit Trusted Gateway", func(t *testing.T) { + t.Parallel() + trustedTests(t, "trusted.com") + }) + + t.Run("Implicit Default Trustless Gateway", func(t *testing.T) { + t.Parallel() + trustlessTests(t, "not.configured.com") + trustlessTests(t, "localhost") + trustlessTests(t, "127.0.0.1") + trustlessTests(t, "::1") + }) +} + +func TestIpnsTrustlessMode(t *testing.T) { + api, root := newMockAPI(t) + api.namesys["/ipns/trustless.com"] = path.FromCid(root) + api.namesys["/ipns/trusted.com"] = path.FromCid(root) + + ts := newTestServerWithConfig(t, api, Config{ + Headers: map[string][]string{}, + NoDNSLink: false, + PublicGateways: map[string]*Specification{ + "trustless.com": { + Paths: []string{"/ipfs", "/ipns"}, + }, + "trusted.com": { + Paths: []string{"/ipfs", "/ipns"}, + DeserializedResponses: true, + }, + }, + }) + t.Logf("test server url: %s", ts.URL) + + doRequest := func(t *testing.T, path, host string, expectedStatus int) { + req, err := http.NewRequest(http.MethodGet, ts.URL+path, nil) + assert.Nil(t, err) + + if host != "" { + req.Host = host + } + + res, err := doWithoutRedirect(req) + assert.Nil(t, err) + defer res.Body.Close() + assert.Equal(t, expectedStatus, res.StatusCode) + } + + // DNSLink only. Not supported for trustless. Supported for trusted, except + // format=ipns-record which is unavailable for DNSLink. + doRequest(t, "/", "trustless.com", http.StatusNotAcceptable) + doRequest(t, "/EmptyDir/", "trustless.com", http.StatusNotAcceptable) + doRequest(t, "/?format=ipns-record", "trustless.com", http.StatusNotAcceptable) + + doRequest(t, "/", "trusted.com", http.StatusOK) + doRequest(t, "/EmptyDir/", "trusted.com", http.StatusOK) + doRequest(t, "/?format=ipns-record", "trusted.com", http.StatusBadRequest) +} diff --git a/gateway/handler.go b/gateway/handler.go index fcb61e416..f996f5648 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -242,6 +242,13 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { i.addUserHeaders(w) // ok, _now_ write user's headers. w.Header().Set("X-Ipfs-Path", contentPath.String()) + // Fail fast if unsupported request type was sent to a Trustless Gateway. + if !i.isDeserializedResponsePossible(r) && !i.isTrustlessRequest(contentPath, responseFormat) { + err := errors.New("only trustless requests are accepted on this gateway: https://specs.ipfs.tech/http-gateways/trustless-gateway/") + webError(w, err, http.StatusNotAcceptable) + return + } + // TODO: Why did the previous code do path resolution, was that a bug? // TODO: Does If-None-Match apply here? if responseFormat == "application/vnd.ipfs.ipns-record" { @@ -325,6 +332,75 @@ func (i *handler) addUserHeaders(w http.ResponseWriter) { } } +// isDeserializedResponsePossible returns true if deserialized responses +// are allowed on the specified hostname, or globally. Host-specific rules +// override global config. +func (i *handler) isDeserializedResponsePossible(r *http.Request) bool { + // Get the value from HTTP Host header + host := r.Host + + // If this request went through WithHostname, use the key in the context. + if h, ok := r.Context().Value(GatewayHostnameKey).(string); ok { + host = h + } + + // If a reverse-proxy passed explicit hostname override + // in the X-Forwarded-Host header, it takes precedence above everything else. + if xHost := r.Header.Get("X-Forwarded-Host"); xHost != "" { + host = xHost + } + + // If the gateway is defined, return whatever is set. + if gw, ok := i.config.PublicGateways[host]; ok { + return gw.DeserializedResponses + } + + // Otherwise, the default. + return i.config.DeserializedResponses +} + +// isTrustlessRequest returns true if the responseFormat and contentPath allow +// client to trustlessly verify response. Relevant response formats are defined +// in the [Trustless Gateway] spec. +// +// [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/ +func (i *handler) isTrustlessRequest(contentPath ipath.Path, responseFormat string) bool { + // Only allow "/{#1}/{#2}"-like paths. + trimmedPath := strings.Trim(contentPath.String(), "/") + pathComponents := strings.Split(trimmedPath, "/") + if len(pathComponents) != 2 { + return false + } + + if contentPath.Namespace() == "ipns" { + // TODO: only ipns records allowed until https://github.com/ipfs/specs/issues/369 is resolved + if responseFormat != "application/vnd.ipfs.ipns-record" { + return false + } + + // Only valid, cryptographically verifiable IPNS record names (no DNSLink on trustless gateways) + // TODO: replace with ipns.Name as part of https://github.com/ipfs/specs/issues/376 + if _, err := peer.Decode(pathComponents[1]); err != nil { + return false + } + + return true + } + + // Only valid CIDs. + if _, err := cid.Decode(pathComponents[1]); err != nil { + return false + } + + switch responseFormat { + case "application/vnd.ipld.raw", + "application/vnd.ipld.car": + return true + default: + return false + } +} + func panicHandler(w http.ResponseWriter) { if r := recover(); r != nil { log.Error("A panic occurred in the gateway handler!") diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index 3747d85d6..5144b3abf 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -210,10 +210,10 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat } func hasOriginIsolation(r *http.Request) bool { - _, gw := r.Context().Value(GatewayHostnameKey).(string) + _, subdomainGw := r.Context().Value(SubdomainHostnameKey).(string) _, dnslink := r.Context().Value(DNSLinkHostnameKey).(string) - if gw || dnslink { + if subdomainGw || dnslink { return true } diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index 801503a5c..1ab45d2dd 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -186,8 +186,10 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * // for this request. var gwURL string - // Get gateway hostname and build gateway URL. - if h, ok := r.Context().Value(GatewayHostnameKey).(string); ok { + // Ensure correct URL in DNSLink and Subdomain Gateways. + if h, ok := r.Context().Value(SubdomainHostnameKey).(string); ok { + gwURL = "//" + h + } else if h, ok := r.Context().Value(DNSLinkHostnameKey).(string); ok { gwURL = "//" + h } else { gwURL = "" diff --git a/gateway/hostname.go b/gateway/hostname.go index bb0d4da25..7c72787b6 100644 --- a/gateway/hostname.go +++ b/gateway/hostname.go @@ -16,49 +16,11 @@ import ( mbase "github.com/multiformats/go-multibase" ) -// Specification is the specification of an IPFS Public Gateway. -type Specification struct { - // Paths is explicit list of path prefixes that should be handled by - // this gateway. Example: `["/ipfs", "/ipns"]` - // Useful if you only want to support immutable `/ipfs`. - Paths []string - - // UseSubdomains indicates whether or not this gateway uses subdomains - // for IPFS resources instead of paths. That is: http://CID.ipfs.GATEWAY/... - // - // If this flag is set, any /ipns/$id and/or /ipfs/$id paths in Paths - // will be permanently redirected to http://$id.[ipns|ipfs].$gateway/. - // - // We do not support using both paths and subdomains for a single domain - // for security reasons (Origin isolation). - UseSubdomains bool - - // NoDNSLink configures this gateway to _not_ resolve DNSLink for the - // specific FQDN provided in `Host` HTTP header. Useful when you want to - // explicitly allow or refuse hosting a single hostname. To refuse all - // DNSLinks in `Host` processing, pass noDNSLink to `WithHostname` instead. - // This flag overrides the global one. - NoDNSLink bool - - // InlineDNSLink configures this gateway to always inline DNSLink names - // (FQDN) into a single DNS label in order to interop with wildcard TLS certs - // and Origin per CID isolation provided by rules like https://publicsuffix.org - // This should be set to true if you use HTTPS. - InlineDNSLink bool -} - // WithHostname is a middleware that can wrap an http.Handler in order to parse the // Host header and translating it to the content path. This is useful for Subdomain // and DNSLink gateways. -// -// publicGateways configures the behavior of known public gateways. Each key is a -// fully qualified domain name (FQDN). -// -// noDNSLink configures the gateway to _not_ perform DNS TXT record lookups in -// response to requests with values in `Host` HTTP header. This flag can be overridden -// per FQDN in publicGateways. -func WithHostname(next http.Handler, api IPFSBackend, publicGateways map[string]*Specification, noDNSLink bool) http.HandlerFunc { - gateways := prepareHostnameGateways(publicGateways) +func WithHostname(c Config, api IPFSBackend, next http.Handler) http.HandlerFunc { + gateways := prepareHostnameGateways(c.PublicGateways) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer panicHandler(w) @@ -112,7 +74,7 @@ func WithHostname(next http.Handler, api IPFSBackend, publicGateways map[string] // Not a subdomain resource, continue with path processing // Example: 127.0.0.1:8080/ipfs/{CID}, ipfs.io/ipfs/{CID} etc - next.ServeHTTP(w, r) + next.ServeHTTP(w, withHostnameContext(r, host)) return } // Not a whitelisted path @@ -121,7 +83,7 @@ func WithHostname(next http.Handler, api IPFSBackend, publicGateways map[string] if !gw.NoDNSLink && hasDNSLinkRecord(r.Context(), api, host) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path - next.ServeHTTP(w, withHostnameContext(r, host)) + next.ServeHTTP(w, withDNSLinkContext(r, host)) return } @@ -219,7 +181,7 @@ func WithHostname(next http.Handler, api IPFSBackend, publicGateways map[string] r.URL.Path = pathPrefix + r.URL.Path // Serve path request - next.ServeHTTP(w, withHostnameContext(r, gwHostname)) + next.ServeHTTP(w, withSubdomainContext(r, gwHostname)) return } @@ -229,11 +191,10 @@ func WithHostname(next http.Handler, api IPFSBackend, publicGateways map[string] // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? // 2. does Host header include a fully qualified domain name (FQDN)? // 3. does DNSLink record exist in DNS? - if !noDNSLink && hasDNSLinkRecord(r.Context(), api, host) { + if !c.NoDNSLink && hasDNSLinkRecord(r.Context(), api, host) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path - ctx := context.WithValue(r.Context(), DNSLinkHostnameKey, host) - next.ServeHTTP(w, withHostnameContext(r.WithContext(ctx), host)) + next.ServeHTTP(w, withDNSLinkContext(r, host)) return } @@ -243,14 +204,23 @@ func WithHostname(next http.Handler, api IPFSBackend, publicGateways map[string] }) } -// Extends request context to include hostname of a canonical gateway root -// (subdomain root or dnslink fqdn) +// withDNSLinkContext extends the context to include the hostname of the DNSLink +// Gateway (https://specs.ipfs.tech/http-gateways/dnslink-gateway/). +func withDNSLinkContext(r *http.Request, hostname string) *http.Request { + ctx := context.WithValue(r.Context(), DNSLinkHostnameKey, hostname) + return withHostnameContext(r.WithContext(ctx), hostname) +} + +// withSubdomainContext extends the context to include the hostname of the +// Subdomain Gateway (https://specs.ipfs.tech/http-gateways/subdomain-gateway/). +func withSubdomainContext(r *http.Request, hostname string) *http.Request { + ctx := context.WithValue(r.Context(), SubdomainHostnameKey, hostname) + return withHostnameContext(r.WithContext(ctx), hostname) +} + +// withHostnameContext extends the context to include the canonical gateway root, +// which can be a Subdomain Gateway, a DNSLink Gateway, or just a regular gateway. func withHostnameContext(r *http.Request, hostname string) *http.Request { - // This is required for links on directory listing pages to work correctly - // on subdomain and dnslink gateways. While DNSlink could read value from - // Host header, subdomain gateways have more comples rules (knownSubdomainDetails) - // More: https://github.com/ipfs/dir-index-html/issues/42 - // nolint: staticcheck // non-backward compatible change ctx := context.WithValue(r.Context(), GatewayHostnameKey, hostname) return r.WithContext(ctx) }