feat(gateway): _redirects file support

core/coreapi/name.go

@@ -37,6 +37,8 @@ func (e *ipnsEntry) Value() path.Path {
 	return e.value
 }
 
+type requestContextKey string
+
 // Publish announces new IPNS name and returns the new IPNS entry.
 func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.NamePublishOption) (coreiface.IpnsEntry, error) {
 	ctx, span := tracing.Span(ctx, "CoreAPI.NameAPI", "Publish", trace.WithAttributes(attribute.String("path", p.String())))
@@ -76,7 +78,7 @@ func (api *NameAPI) Publish(ctx context.Context, p path.Path, opts ...caopts.Nam
 
 	if options.TTL != nil {
 		// nolint: staticcheck // non-backward compatible change
-		ctx = context.WithValue(ctx, "ipns-publish-ttl", *options.TTL)
+		ctx = context.WithValue(ctx, requestContextKey("ipns-publish-ttl"), *options.TTL)
 	}
 
 	eol := time.Now().Add(options.ValidTime)

core/corehttp/gateway_handler.go

@@ -24,7 +24,6 @@ import (
 	mfs "github.com/ipfs/go-mfs"
 	path "github.com/ipfs/go-path"
 	"github.com/ipfs/go-path/resolver"
-	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	routing "github.com/libp2p/go-libp2p/core/routing"
 	prometheus "github.com/prometheus/client_golang/prometheus"
@@ -378,30 +377,18 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	// Resolve path to the final DAG node for the ETag
-	resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath)
-	switch err {
-	case nil:
-	case coreiface.ErrOffline:
-		webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable)
-		return
-	default:
-		// if Accept is text/html, see if ipfs-404.html is present
-		if i.servePretty404IfPresent(w, r, contentPath) {
-			logger.Debugw("serve pretty 404 if present")
-			return
-		}
-		webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusBadRequest)
-		return
-	}
-
 	// Detect when explicit Accept header or ?format parameter are present
 	responseFormat, formatParams, err := customResponseFormat(r)
 	if err != nil {
 		webError(w, "error while processing the Accept header", err, http.StatusBadRequest)
 		return
 	}
 	trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat))
+
+	resolvedPath, contentPath, ok := i.handlePathResolution(w, r, responseFormat, contentPath, logger)
+	if !ok {
+		return
+	}
 	trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResolvedPath", resolvedPath.String()))
 
 	// Detect when If-None-Match HTTP header allows returning HTTP 304 Not Modified
@@ -450,7 +437,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 	}
 }
 
-func (i *gatewayHandler) servePretty404IfPresent(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool {
+// Deprecated: legacy ipfs-404.html files are superseded by _redirects file
+// This is provided only for backward-compatibility, until websites migrate
+// to 404s managed via _redirects file (https://github.com/ipfs/specs/pull/290)
+func (i *gatewayHandler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool {
 	resolved404Path, ctype, err := i.searchUpTreeFor404(r, contentPath)
 	if err != nil {
 		return false

core/corehttp/gateway_handler_unixfs__redirects.go

@@ -0,0 +1,260 @@
+package corehttp
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	gopath "path"
+	"strconv"
+	"strings"
+
+	files "github.com/ipfs/go-ipfs-files"
+	redirects "github.com/ipfs/go-ipfs-redirects-file"
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
+	ipath "github.com/ipfs/interface-go-ipfs-core/path"
+	"go.uber.org/zap"
+)
+
+// Resolve the provided path.
+func (i *gatewayHandler) handlePathResolution(w http.ResponseWriter, r *http.Request, responseFormat string, contentPath ipath.Path, logger *zap.SugaredLogger) (ipath.Resolved, ipath.Path, bool) {
+	// Attempt to resolve the provided path.
+	resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath)
+
+	switch err {
+	case nil:
+		return resolvedPath, contentPath, true
+	case coreiface.ErrOffline:
+		webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable)
+		return nil, nil, false
+	default:
+		if isUnixfsResponseFormat(responseFormat) {
+			// The path can't be resolved.
+			// If we have origin isolation, attempt to handle any redirect rules.
+			if hasOriginIsolation(r) {
+				resolvedPath, contentPath, ok, hadMatchingRule := i.serveRedirectsIfPresent(w, r, resolvedPath, contentPath, logger)
+				if hadMatchingRule {
+					return resolvedPath, contentPath, ok
+				}
+			}
+
+			// if Accept is text/html, see if ipfs-404.html is present
+			// This logic isn't documented and will likely be removed at some point.
+			// Any 404 logic in _redirects above will have already run by this time, so it's really an extra fall back
+			if i.serveLegacy404IfPresent(w, r, contentPath) {
+				logger.Debugw("serve pretty 404 if present")
+				return nil, nil, false
+			}
+
+			// Fallback
+			webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusBadRequest)
+			return nil, nil, false
+		} else {
+			webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound)
+			return nil, nil, false
+		}
+	}
+}
+
+// Resolving a UnixFS path involves determining if the provided `path.Path` exists and returning the `path.Resolved`
+// corresponding to that path. For UnixFS, path resolution is more involved if a `_redirects` file exists, stored
+// underneath the root CID of the path.
+//
+// Example 1:
+// If a path exists, we always return the `path.Resolved` corresponding to that path, regardless of the existence of a `_redirects` file.
+//
+// Example 2:
+// If a path does not exist, usually we should return a `nil` resolution path and an error indicating that the path
+// doesn't exist.  However, a `_redirects` file may exist and contain a redirect rule that redirects that path to a different path.
+// We need to evaluate the rule and perform the redirect if present.
+//
+// Example 3:
+// Another possibility is that the path corresponds to a rewrite rule (i.e. a rule with a status of 200).
+// In this case, we don't perform a redirect, but do need to return a `path.Resolved` and `path.Path` corresponding to
+// the rewrite destination path.
+//
+// Note that for security reasons, redirect rules are only processed when the request has origin isolation.
+// See https://github.com/ipfs/specs/pull/290 for more information.
+func (i *gatewayHandler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, logger *zap.SugaredLogger) (newResolvedPath ipath.Resolved, newContentPath ipath.Path, ok bool, hadMatchingRule bool) {
+	redirectsFile := i.getRedirectsFile(r, contentPath, logger)
+	if redirectsFile != nil {
+		redirectRules, err := i.getRedirectRules(r, redirectsFile)
+		if err != nil {
+			internalWebError(w, err)
+			return nil, nil, false, true
+		}
+
+		redirected, newPath, err := i.handleRedirectsFileRules(w, r, contentPath, redirectRules)
+		if err != nil {
+			err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsFile.String(), err)
+			internalWebError(w, err)
+			return nil, nil, false, true
+		}
+
+		if redirected {
+			return nil, nil, false, true
+		}
+
+		// 200 is treated as a rewrite, so update the path and continue
+		if newPath != "" {
+			// Reassign contentPath and resolvedPath since the URL was rewritten
+			contentPath = ipath.New(newPath)
+			resolvedPath, err = i.api.ResolvePath(r.Context(), contentPath)
+			if err != nil {
+				internalWebError(w, err)
+				return nil, nil, false, true
+			}
+
+			return resolvedPath, contentPath, true, true
+		}
+	}
+	// No matching rule
+	return resolvedPath, contentPath, false, false
+}
+
+func (i *gatewayHandler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, redirectRules []redirects.Rule) (bool, string, error) {
+	// Attempt to match a rule to the URL path, and perform the corresponding redirect or rewrite
+	pathParts := strings.Split(contentPath.String(), "/")
+	if len(pathParts) > 3 {
+		// All paths should start with /ipfs/cid/, so get the path after that
+		urlPath := "/" + strings.Join(pathParts[3:], "/")
+		rootPath := strings.Join(pathParts[:3], "/")
+		// Trim off the trailing /
+		urlPath = strings.TrimSuffix(urlPath, "/")
+
+		for _, rule := range redirectRules {
+			// Error right away if the rule is invalid
+			if !rule.MatchAndExpandPlaceholders(urlPath) {
+				continue
+			}
+
+			// We have a match!
+
+			// Rewrite
+			if rule.Status == 200 {
+				// Prepend the rootPath
+				toPath := rootPath + rule.To
+				return false, toPath, nil
+			}
+
+			// Or 400s
+			if rule.Status == 404 {
+				toPath := rootPath + rule.To
+				content404Path := ipath.New(toPath)
+				err := i.serve404(w, r, content404Path)
+				return true, toPath, err
+			}
+
+			if rule.Status == 410 {
+				webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), coreiface.ErrResolveFailed, http.StatusGone)
+				return true, rule.To, nil
+			}
+
+			if rule.Status == 451 {
+				webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), coreiface.ErrResolveFailed, http.StatusUnavailableForLegalReasons)
+				return true, rule.To, nil
+			}
+
+			// Or redirect
+			if rule.Status >= 301 && rule.Status <= 308 {
+				http.Redirect(w, r, rule.To, rule.Status)
+				return true, rule.To, nil
+			}
+		}
+	}
+
+	// No redirects matched
+	return false, "", nil
+}
+
+func (i *gatewayHandler) getRedirectRules(r *http.Request, redirectsFilePath ipath.Resolved) ([]redirects.Rule, error) {
+	// Convert the path into a file node
+	node, err := i.api.Unixfs().Get(r.Context(), redirectsFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("could not get _redirects node: %v", err)
+	}
+	defer node.Close()
+
+	// Convert the node into a file
+	f, ok := node.(files.File)
+	if !ok {
+		return nil, fmt.Errorf("could not parse _redirects: %v", err)
+	}
+
+	// Parse redirect rules from file
+	redirectRules, err := redirects.Parse(f)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse _redirects: %v", err)
+	}
+
+	return redirectRules, nil
+}
+
+// Returns a resolved path to the _redirects file located in the root CID path of the requested path
+func (i *gatewayHandler) getRedirectsFile(r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) ipath.Resolved {
+	// contentPath is the full ipfs path to the requested resource,
+	// regardless of whether path or subdomain resolution is used.
+	rootPath := getRootPath(contentPath)
+
+	// Check for _redirects file.
+	// Any path resolution failures are ignored and we just assume there's no _redirects file.
+	// Note that ignoring these errors also ensures that the use of the empty CID (bafkqaaa) in tests doesn't fail.
+	path := ipath.Join(rootPath, "_redirects")
+	resolvedPath, err := i.api.ResolvePath(r.Context(), path)
+	if err != nil {
+		return nil
+	}
+	return resolvedPath
+}
+
+// Returns the root CID Path for the given path
+func getRootPath(path ipath.Path) ipath.Path {
+	parts := strings.Split(path.String(), "/")
+	return ipath.New(gopath.Join("/", path.Namespace(), parts[2]))
+}
+
+func (i *gatewayHandler) serve404(w http.ResponseWriter, r *http.Request, content404Path ipath.Path) error {
+	resolved404Path, err := i.api.ResolvePath(r.Context(), content404Path)
+	if err != nil {
+		return err
+	}
+
+	node, err := i.api.Unixfs().Get(r.Context(), resolved404Path)
+	if err != nil {
+		return err
+	}
+	defer node.Close()
+
+	f, ok := node.(files.File)
+	if !ok {
+		return fmt.Errorf("could not convert node for 404 page to file")
+	}
+
+	size, err := f.Size()
+	if err != nil {
+		return fmt.Errorf("could not get size of 404 page")
+	}
+
+	log.Debugw("using _redirects 404 file", "path", content404Path)
+	w.Header().Set("Content-Type", "text/html")
+	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
+	addCacheControlHeaders(w, r, content404Path, resolved404Path.Cid())
+	w.WriteHeader(http.StatusNotFound)
+	_, err = io.CopyN(w, f, size)
+	return err
+}
+
+func hasOriginIsolation(r *http.Request) bool {
+	_, gw := r.Context().Value(requestContextKey("gw-hostname")).(string)
+	_, dnslink := r.Context().Value("dnslink-hostname").(string)
+
+	if gw || dnslink {
+		return true
+	}
+
+	return false
+}
+
+func isUnixfsResponseFormat(responseFormat string) bool {
+	// The implicit response format is UnixFS
+	return responseFormat == ""
+}

core/corehttp/gateway_handler_unixfs_dir.go

@@ -185,7 +185,7 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 	var gwURL string
 
 	// Get gateway hostname and build gateway URL.
-	if h, ok := r.Context().Value("gw-hostname").(string); ok {
+	if h, ok := r.Context().Value(requestContextKey("gw-hostname")).(string); ok {
 		gwURL = "//" + h
 	} else {
 		gwURL = ""

core/corehttp/hostname.go

@@ -221,7 +221,8 @@ func HostnameOption() ServeOption {
 			if !cfg.Gateway.NoDNSLink && isDNSLinkName(r.Context(), coreAPI, host) {
 				// rewrite path and handle as DNSLink
 				r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path
-				childMux.ServeHTTP(w, withHostnameContext(r, host))
+				ctx := context.WithValue(r.Context(), requestContextKey("dnslink-hostname"), host)
+				childMux.ServeHTTP(w, withHostnameContext(r.WithContext(ctx), host))
 				return
 			}
 
@@ -242,6 +243,8 @@ type wildcardHost struct {
 	spec *config.GatewaySpec
 }
 
+type requestContextKey string
+
 // Extends request context to include hostname of a canonical gateway root
 // (subdomain root or dnslink fqdn)
 func withHostnameContext(r *http.Request, hostname string) *http.Request {
@@ -250,7 +253,7 @@ func withHostnameContext(r *http.Request, hostname string) *http.Request {
 	// Host header, subdomain gateways have more comples rules (knownSubdomainDetails)
 	// More: https://github.com/ipfs/dir-index-html/issues/42
 	// nolint: staticcheck // non-backward compatible change
-	ctx := context.WithValue(r.Context(), "gw-hostname", hostname)
+	ctx := context.WithValue(r.Context(), requestContextKey("gw-hostname"), hostname)
 	return r.WithContext(ctx)
 }
 

docs/examples/kubo-as-a-library/go.sum

@@ -571,6 +571,7 @@ github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY
 github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
 github.com/ipfs/go-ipfs-provider v0.7.1 h1:eKToBUAb6ZY8iiA6AYVxzW4G1ep67XUaaEBUIYpxhfw=
 github.com/ipfs/go-ipfs-provider v0.7.1/go.mod h1:QwdDYRYnC5sYGLlOwVDY/0ZB6T3zcMtu+5+GdGeUuw8=
+github.com/ipfs/go-ipfs-redirects-file v0.1.0/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk=
 github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs=
 github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=
 github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FFiW0BY=
@@ -1535,9 +1536,11 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
+github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -2179,6 +2182,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=