From 2cecd47bac56c4972a763cffeeb5d1de3c98c93e Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 15 Feb 2024 00:18:32 +0100 Subject: [PATCH] feat: trustless-only mode (RAINBOW_TRUSTLESS_GATEWAY_DOMAINS) (#81) * feat: trustless gateway option * docs: RAINBOW_TRUSTLESS_GATEWAY_DOMAINS * Apply suggestions from code review * test: add trustless e2e test * feat: print gateway domains when set --------- Co-authored-by: Marcin Rataj Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com> --- docs/environment-variables.md | 43 ++++++++++++-- go.mod | 3 +- handler_test.go | 109 ++++++++++++++++++++++++++++++++++ handlers.go | 20 +++++++ main.go | 35 ++++++++--- setup.go | 1 + 6 files changed, 195 insertions(+), 16 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 6e2d9d7..29f890b 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -5,6 +5,7 @@ - [Configuration](#configuration) - [`RAINBOW_GATEWAY_DOMAINS`](#rainbow_gateway_domains) - [`RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS`](#rainbow_subdomain_gateway_domains) + - [`RAINBOW_TRUSTLESS_GATEWAY_DOMAINS`](#rainbow_trustless_gateway_domains) - [`KUBO_RPC_URL`](#kubo_rpc_url) - [Logging](#logging) - [`GOLOG_LOG_LEVEL`](#golog_log_level) @@ -19,26 +20,56 @@ ### `RAINBOW_GATEWAY_DOMAINS` -Comma-separated list of path gateway hostnames. For example, passing `ipfs.io` will enable handler for standard [path gateway](https://specs.ipfs.tech/http-gateways/path-gateway/) requests with the `Host` header set to `ipfs.io`. +Comma-separated list of [path gateway](https://specs.ipfs.tech/http-gateways/path-gateway/) +hostnames that will serve both trustless and deserialized response types. -Default: `127.0.0.1` +Example: passing `ipfs.io` will enable deserialized handler for flat +[path gateway](https://specs.ipfs.tech/http-gateways/path-gateway/) +requests with the `Host` header set to `ipfs.io`. +Default: `127.0.0.1` ### `RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS` -Comma-separated list of [subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) domains. For example, passing `dweb.link` will enable handler for standard [subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) requests with the `Host` header set to `*.ipfs.dweb.link` and `*.ipns.dweb.link`. +Comma-separated list of [subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) +domains for website hosting with Origin-isolation per content root. + +Example: passing `dweb.link` will enable handler for Origin-isolated +[subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) +requests with the `Host` header with subdomain values matching +`*.ipfs.dweb.link` or `*.ipns.dweb.link`. Default: `localhost` -### `KUBO_RPC_URL` +### `RAINBOW_TRUSTLESS_GATEWAY_DOMAINS` -Default: `127.0.0.1:5001` (see `DefaultKuboRPC`) +Specifies trustless-only hostnames. + +Comma-separated list of [trustless gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) +domains, where unverified website asset hosting and deserialized responses is +disabled, and **response types requested via `?format=` and `Accept` HTTP header are limited to +[verifiable content types](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval)**: +- [`application/vnd.ipld.raw`](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw) +- [`application/vnd.ipld.car`](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) +- [`application/vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record) + +**NOTE:** This setting is applied on top of everything else, to ensure +trustless domains can't be used for phishing or direct hotlinking and hosting of third-party content. Hostnames that are passed to both `RAINBOW_GATEWAY_DOMAINS` and `RAINBOW_TRUSTLESS_GATEWAY_DOMAINS` will work only as trustless gateways. + +Example: passing `trustless-gateway.link` will ensure only verifiable content types are supported +when request comes with the `Host` header set to `trustless-gateway.link`. + +Default: none (`Host` is ignored and gateway at `127.0.0.1` supports both deserialized and verifiable response types) + +### `KUBO_RPC_URL` Single URL or a comma separated list of RPC endpoints that provide legacy `/api/v0` from Kubo. We use this to redirect some legacy `/api/v0` commands that need to be handled on `ipfs.io`. -This is deprecated and will be removed in the future. +**NOTE:** This is deprecated and will be removed in the future. + +Default: `127.0.0.1:5001` (see `DefaultKuboRPC`) ## Logging diff --git a/go.mod b/go.mod index b401299..d2803c7 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/mitchellh/go-server-timing v1.0.1 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.2 + github.com/multiformats/go-multicodec v0.9.0 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/prometheus/client_golang v1.18.0 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 @@ -126,7 +127,6 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect - github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect @@ -153,6 +153,7 @@ require ( github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20240109153615-66e95c3e8a87 // indirect + github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opentelemetry.io/contrib/propagators/aws v1.23.0 // indirect diff --git a/handler_test.go b/handler_test.go index 959bc31..8dd709e 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1,11 +1,22 @@ package main import ( + "bytes" + "context" "net/http" "net/http/httptest" "testing" + chunker "github.com/ipfs/boxo/chunker" + "github.com/ipfs/boxo/ipld/merkledag" + "github.com/ipfs/boxo/ipld/unixfs/importer/balanced" + uih "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" + util "github.com/ipfs/boxo/util" + "github.com/ipfs/go-cid" + ic "github.com/libp2p/go-libp2p/core/crypto" + "github.com/multiformats/go-multicodec" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type rpcRedirectTest struct { @@ -85,3 +96,101 @@ func TestRPCNotImplemented(t *testing.T) { assert.Equal(t, http.StatusNotImplemented, resp.Code) } } + +func mustTestServer(t *testing.T, cfg Config) (*httptest.Server, *Node) { + cfg.DataDir = t.TempDir() + cfg.BlockstoreType = "flatfs" + + ctx := context.Background() + + sr := util.NewTimeSeededRand() + sk, _, err := ic.GenerateKeyPairWithReader(ic.Ed25519, 2048, sr) + require.NoError(t, err) + + cdns := newCachedDNS(dnsCacheRefreshInterval) + + t.Cleanup(func() { + _ = cdns.Close() + }) + + gnd, err := Setup(ctx, cfg, sk, cdns) + if err != nil { + require.NoError(t, err) + } + + handler, err := setupGatewayHandler(cfg, gnd) + if err != nil { + require.NoError(t, err) + } + + ts := httptest.NewServer(handler) + + return ts, gnd +} + +func mustAddFile(t *testing.T, gnd *Node, content []byte) cid.Cid { + dsrv := merkledag.NewDAGService(gnd.bsrv) + + // Create a UnixFS graph from our file, parameters described here but can be visualized at https://dag.ipfs.tech/ + ufsImportParams := uih.DagBuilderParams{ + Maxlinks: uih.DefaultLinksPerBlock, // Default max of 174 links per block + RawLeaves: true, // Leave the actual file bytes untouched instead of wrapping them in a dag-pb protobuf wrapper + CidBuilder: cid.V1Builder{ // Use CIDv1 for all links + Codec: uint64(multicodec.DagPb), + MhType: uint64(multicodec.Sha2_256), // Use SHA2-256 as the hash function + MhLength: -1, // Use the default hash length for the given hash function (in this case 256 bits) + }, + Dagserv: dsrv, + NoCopy: false, + } + ufsBuilder, err := ufsImportParams.New(chunker.NewSizeSplitter(bytes.NewReader(content), chunker.DefaultBlockSize)) // Split the file up into fixed sized 256KiB chunks + require.NoError(t, err) + + nd, err := balanced.Layout(ufsBuilder) // Arrange the graph with a balanced layout + require.NoError(t, err) + + return nd.Cid() +} + +func TestTrustless(t *testing.T) { + t.Parallel() + + ts, gnd := mustTestServer(t, Config{ + TrustlessGatewayDomains: []string{"trustless.com"}, + }) + + content := "hello world" + cid := mustAddFile(t, gnd, []byte(content)) + url := ts.URL + "/ipfs/" + cid.String() + + t.Run("Non-trustless request returns 406", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + req.Host = "trustless.com" + + res, err := http.DefaultClient.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + }) + + t.Run("Trustless request with query parameter returns 200", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, url+"?format=raw", nil) + require.NoError(t, err) + req.Host = "trustless.com" + + res, err := http.DefaultClient.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + }) + + t.Run("Trustless request with accept header returns 200", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + req.Host = "trustless.com" + req.Header.Set("Accept", "application/vnd.ipld.raw") + + res, err := http.DefaultClient.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + }) +} diff --git a/handlers.go b/handlers.go index 2178d82..bdced19 100644 --- a/handlers.go +++ b/handlers.go @@ -135,6 +135,16 @@ func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) { } } + for _, domain := range cfg.TrustlessGatewayDomains { + publicGateways[domain] = &gateway.PublicGateway{ + Paths: []string{"/ipfs", "/ipns", "/version"}, + NoDNSLink: true, + InlineDNSLink: true, + DeserializedResponses: false, + UseSubdomains: contains(cfg.SubdomainGatewayDomains, domain), + } + } + // If we're doing tests, ensure the right public gateways are enabled. if os.Getenv("GATEWAY_CONFORMANCE_TEST") == "true" { publicGateways["example.com"] = &gateway.PublicGateway{ @@ -340,3 +350,13 @@ func BlockProfileRateOption(path string, mux *http.ServeMux) *http.ServeMux { }) return mux } + +func contains[T comparable](collection []T, element T) bool { + for _, item := range collection { + if item == element { + return true + } + } + + return false +} diff --git a/main.go b/main.go index e1fb2a4..8b909a8 100644 --- a/main.go +++ b/main.go @@ -46,12 +46,12 @@ only websites, but any supported content-addressed Merkle-DAG), in formats that are suitable for verification client-side (i.e. CAR files). Rainbow is optimized to perform the tasks of a gateway and only that, making -opinionated choices on the configration and setup of internal +opinionated choices on the configuration and setup of internal components. Rainbow aims to serve production environments, where gateways are deployed as a public service meant to be accessible by anyone. Rainbow acts as a client to the IPFS network and does not serve or provide content to it. Rainbow cannot be used to store or pin IPFS content, other than that -temporailly served over HTTP. Rainbow is just a gateway. +temporarily served over HTTP. Rainbow is just a gateway. Persistent configuration and data is stored in $RAINBOW_DATADIR (by default, the folder in which rainbow is run). @@ -73,7 +73,6 @@ Generate an identity seed and launch a gateway: ` app.Flags = []cli.Flag{ - &cli.StringFlag{ Name: "datadir", Value: "", @@ -96,13 +95,19 @@ Generate an identity seed and launch a gateway: Name: "gateway-domains", Value: "", EnvVars: []string{"RAINBOW_GATEWAY_DOMAINS"}, - Usage: "Legacy path-gateway domains. Comma-separated list.", + Usage: "Domains with flat path gateway, no Origin isolation. Comma-separated list.", }, &cli.StringFlag{ Name: "subdomain-gateway-domains", Value: "", EnvVars: []string{"RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS"}, - Usage: "Subdomain gateway domains. Comma-separated list.", + Usage: "Domains with subdomain-based Origin isolation. Comma-separated list.", + }, + &cli.StringFlag{ + Name: "trustless-gateway-domains", + Value: "", + EnvVars: []string{"RAINBOW_TRUSTLESS_GATEWAY_DOMAINS"}, + Usage: "Domains limited to trustless, verifiable response types. Comma-separated list.", }, &cli.StringFlag{ Name: "gateway-listen-address", @@ -116,7 +121,6 @@ Generate an identity seed and launch a gateway: EnvVars: []string{"RAINBOW_CTL_LISTEN_ADDRESS"}, Usage: "Listen address for the management api and metrics", }, - &cli.IntFlag{ Name: "connmgr-low", Value: 100, @@ -270,6 +274,7 @@ share the same seed as long as the indexes are different. BlockstoreType: cctx.String("blockstore"), GatewayDomains: getCommaSeparatedList(cctx.String("gateway-domains")), SubdomainGatewayDomains: getCommaSeparatedList(cctx.String("subdomain-gateway-domains")), + TrustlessGatewayDomains: getCommaSeparatedList(cctx.String("trustless-gateway-domains")), ConnMgrLow: cctx.Int("connmgr-low"), ConnMgrHi: cctx.Int("connmgr-high"), ConnMgrGrace: cctx.Duration("connmgr-grace"), @@ -334,10 +339,16 @@ share the same seed as long as the indexes are different. var wg sync.WaitGroup wg.Add(2) - fmt.Printf("Gateway listening at %s\n", gatewayListen) - fmt.Printf("Legacy RPC at /api/v0 (%s): %s\n", EnvKuboRPC, strings.Join(gnd.kuboRPCs, " ")) + fmt.Printf("IPFS Gateway listening at %s\n\n", gatewayListen) + + printIfListConfigured(" RAINBOW_GATEWAY_DOMAINS = ", cfg.GatewayDomains) + printIfListConfigured(" RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS = ", cfg.SubdomainGatewayDomains) + printIfListConfigured(" RAINBOW_TRUSTLESS_GATEWAY_DOMAINS = ", cfg.TrustlessGatewayDomains) + printIfListConfigured(" Legacy RPC at /api/v0 will redirect to KUBO_RPC_URL = ", cfg.KuboRPCURLs) + + fmt.Printf("\n") fmt.Printf("CTL endpoint listening at http://%s\n", ctlListen) - fmt.Printf("Metrics: http://%s/debug/metrics/prometheus\n\n", ctlListen) + fmt.Printf(" Metrics: http://%s/debug/metrics/prometheus\n\n", ctlListen) go func() { defer wg.Done() @@ -424,3 +435,9 @@ func getCommaSeparatedList(val string) []string { } return items } + +func printIfListConfigured(message string, list []string) { + if len(list) > 0 { + fmt.Printf(message+"%v\n", strings.Join(list, ", ")) + } +} diff --git a/setup.go b/setup.go index b96a90f..efb382f 100644 --- a/setup.go +++ b/setup.go @@ -96,6 +96,7 @@ type Config struct { GatewayDomains []string SubdomainGatewayDomains []string + TrustlessGatewayDomains []string RoutingV1 string KuboRPCURLs []string DHTSharedHost bool