feat: remote backend (#117)

.github/workflows/gateway-conformance.yml

@@ -1,42 +1,52 @@
 name: Gateway Conformance
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
   pull_request:
+    paths-ignore:
+      - '**/*.md'
+
+env:
+  GATEWAY_CONFORMANCE_TEST: true  # rainbow preset for conformance testing
+  KUBO_VER: 'v0.28.0'             # kubo daemon used as no-libp2p-remote-* backend
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  gateway-conformance:
+  test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        backend: ["libp2p-bitswap", "remote-block-gw", "remote-car-gw"]
+
     steps:
       # 1. Start the Kubo gateway
-      - name: Setup Go
-        uses: actions/setup-go@v5
+      - name: Install Kubo
+        uses: ipfs/download-ipfs-distribution-action@v1
         with:
-          go-version: 1.21.x
+          name: kubo
+          version: "${{ env.KUBO_VER }}"
 
-      - name: Install Kubo gateway from source
-        #uses: ipfs/download-ipfs-distribution-action@v1
-        run: |
-          go install github.com/ipfs/kubo/cmd/ipfs@v0.24.0-rc1
       - name: Setup kubo config
         run: |
           ipfs init --profile=test
           ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
           ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
           ipfs config --json Gateway.ExposeRoutingAPI true
+          ipfs config Routing.Type "autoclient"
 
       # 2. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
         uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.5.1
         with:
           output: fixtures
 
-      - name: Start Kubo gateway
-        uses: ipfs/start-ipfs-daemon-action@v1
-
-      # 3. Populate the Kubo gateway with the gateway-conformance fixtures
+      # 3. Populate the Kubo node with the gateway-conformance fixtures
       - name: Import fixtures
         run: |
           # Import car files
@@ -56,7 +66,15 @@ jobs:
           export IPFS_NS_MAP="$(cat "./fixtures/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}"
           echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV
 
+      - name: Start Kubo gateway
+        uses: ipfs/start-ipfs-daemon-action@v1
+
       # 4. Build rainbow
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21.x
+
       - name: Checkout rainbow
         uses: actions/checkout@v4
         with:
@@ -65,16 +83,47 @@ jobs:
         run: go build
         working-directory: rainbow
 
-      # 5. Start rainbow
-      - name: Start rainbow
+      # 5. Start rainbow variant
+      - name: Start rainbow (libp2p and bitswap)
+        if: ${{ matrix.backend == 'libp2p-bitswap' }}
         env:
-          GATEWAY_CONFORMANCE_TEST: true
+          RAINBOW_DHT_ROUTING: off
+          RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080
         run: |
-          # get kubo peerID
+          # set up peering with kubo to ensure fixtures can be found fast
           kuboNodeMultiaddr=$(ipfs --api=/ip4/127.0.0.1/tcp/5001 swarm addrs local --id | head -n 1)
 
-          # run gw
-          ./rainbow --http-routers=http://127.0.0.1:8080 --dht-routing=off --peering=$kuboNodeMultiaddr &
+          ./rainbow --peering=$kuboNodeMultiaddr &
+        working-directory: rainbow
+
+      # 5. Start rainbow variant
+      - name: Start rainbow (no libp2p, remote block gateway)
+        if: ${{ matrix.backend == 'remote-block-gw' }}
+        env:
+          RAINBOW_REMOTE_BACKENDS: http://127.0.0.1:8080
+          RAINBOW_REMOTE_BACKENDS_MODE: block
+          RAINBOW_REMOTE_BACKENDS_IPNS: true
+          RAINBOW_LIBP2P: false
+          RAINBOW_BITSWAP: false
+          RAINBOW_DHT_ROUTING: off
+          RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080
+        run: |
+          ./rainbow &
+        working-directory: rainbow
+          #
+      # 5. Start rainbow variant
+      - name: Start rainbow (no libp2p, remote car gateway)
+        if: ${{ matrix.backend == 'remote-car-gw' }}
+        env:
+          RAINBOW_REMOTE_BACKENDS: http://127.0.0.1:8080
+          RAINBOW_REMOTE_BACKENDS_MODE: car
+          RAINBOW_REMOTE_BACKENDS_IPNS: true
+          RAINBOW_LIBP2P: false
+          RAINBOW_BITSWAP: false
+          RAINBOW_DHT_ROUTING: off
+          RAINBOW_HTTP_ROUTERS: http://127.0.0.1:8080
+        run: |
+          ./rainbow &
         working-directory: rainbow
 
       # 6. Run the gateway-conformance tests
@@ -92,7 +141,7 @@ jobs:
           #
           # only-if-cached: rainbow does not guarantee local cache, we will adjust upstream test (which was Kubo-specific)
           # for now disabling these test cases
-          args: -skip 'TestGatewayCache/.*_for_/ipfs/_with_only-if-cached_succeeds_when_in_local_datastore'
+          args: -skip 'TestGatewayCache/.*_with_only-if-cached_succeeds_when_in_local_datastore'
 
       # 7. Upload the results
       - name: Upload MD summary
@@ -102,11 +151,11 @@ jobs:
         if: failure() || success()
         uses: actions/upload-artifact@v4
         with:
-          name: gateway-conformance.html
+          name: ${{ matrix.backend }}_gateway-conformance.html
           path: output.html
       - name: Upload JSON report
         if: failure() || success()
         uses: actions/upload-artifact@v4
         with:
-          name: gateway-conformance.json
+          name: ${{ matrix.backend }}_gateway-conformance.json
           path: output.json

CHANGELOG.md

@@ -15,6 +15,8 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- Now supports remote backends (using RAW block or CAR requests) via `--remote-backends` (`RAINBOW_REMOTE_BACKENDS`).
+
 ### Changed
 
 ### Removed

docs/environment-variables.md

@@ -6,15 +6,22 @@
   - [`RAINBOW_GATEWAY_DOMAINS`](#rainbow_gateway_domains)
   - [`RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS`](#rainbow_subdomain_gateway_domains)
   - [`RAINBOW_TRUSTLESS_GATEWAY_DOMAINS`](#rainbow_trustless_gateway_domains)
+  - [`RAINBOW_DATADIR`](#rainbow_datadir)
   - [`RAINBOW_GC_INTERVAL`](#rainbow_gc_interval)
   - [`RAINBOW_GC_THRESHOLD`](#rainbow_gc_threshold)
   - [`RAINBOW_IPNS_MAX_CACHE_TTL`](#rainbow_ipns_max_cache_ttl)
   - [`RAINBOW_PEERING`](#rainbow_peering)
   - [`RAINBOW_SEED`](#rainbow_seed)
   - [`RAINBOW_SEED_INDEX`](#rainbow_seed_index)
+  - [`RAINBOW_DHT_ROUTING`](#rainbow_dht_routing)
+  - [`RAINBOW_HTTP_ROUTERS`](#rainbow_http_routers)
+- [Experiments](#experiments)
   - [`RAINBOW_SEED_PEERING`](#rainbow_seed_peering)
   - [`RAINBOW_SEED_PEERING_MAX_INDEX`](#rainbow_seed_peering_max_index)
   - [`RAINBOW_PEERING_SHARED_CACHE`](#rainbow_peering_shared_cache)
+  - [`RAINBOW_REMOTE_BACKENDS`](#rainbow_remote_backends)
+  - [`RAINBOW_REMOTE_BACKENDS_MODE`](#rainbow_remote_backends_mode)
+  - [`RAINBOW_REMOTE_BACKENDS_IPNS`](#rainbow_remote_backends_ipns)
 - [Logging](#logging)
   - [`GOLOG_LOG_LEVEL`](#golog_log_level)
   - [`GOLOG_LOG_FMT`](#golog_log_fmt)
@@ -69,6 +76,12 @@ when request comes with the `Host` header set to `trustless-gateway.link`.
 
 Default: none (`Host` is ignored and gateway at `127.0.0.1` supports both deserialized and verifiable response types)
 
+### `RAINBOW_DATADIR`
+
+Directory for persistent data (keys, blocks, denylists)
+
+Default: not set (uses the current directory)
+
 ### `RAINBOW_GC_INTERVAL`
 
 The interval at which the garbage collector will be called. This is given as a string that corresponds to the duration of the interval. Set 0 to disable.
@@ -121,6 +134,20 @@ Index to derivate the PeerID identity from `RAINBOW_SEED`.
 
 Default: not set
 
+### `RAINBOW_DHT_ROUTING`
+
+Control the type of Amino DHT client used for for routing. Options are `accelerated`, `standard` and `off`.
+
+Default: `accelerated`
+
+### `RAINBOW_HTTP_ROUTERS`
+
+HTTP servers with /routing/v1 endpoints to use for delegated routing (comma-separated).
+
+Default: `https://cid.contact`
+
+## Experiments
+
 ### `RAINBOW_SEED_PEERING`
 
 > [!WARNING]
@@ -162,6 +189,34 @@ queries from these safelisted peers, serving locally cached blocks if requested.
 
 Default: `false` (no cache sharing)
 
+### `RAINBOW_REMOTE_BACKENDS`
+
+> [!WARNING]
+> Experimental feature, forces setting `RAINBOW_LIBP2P=false`.
+
+URL(s) of of remote [trustless gateways](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval)
+to use as backend instead of libp2p node with Bitswap.
+
+Default: not set
+
+### `RAINBOW_REMOTE_BACKENDS_MODE`
+
+Requires `RAINBOW_REMOTE_BACKENDS` to be set.
+
+Controls how requests to remote backend are made.
+
+- `block`  will use [application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw) to fetch raw blocks one by one
+- `car` will use [application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) and [IPIP-402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/) for fetching multiple blocks per request
+
+Default: `block`
+
+### `RAINBOW_REMOTE_BACKENDS_IPNS`
+
+Controls whether to fetch IPNS Records ([`application/vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record)) from trustless gateway defined in `RAINBOW_REMOTE_BACKENDS`.
+This is done in addition to other routing systems, such as `RAINBOW_DHT_ROUTING` or `RAINBOW_HTTP_ROUTERS` (if also enabled).
+
+Default: `true`
+
 ## Logging
 
 ### `GOLOG_LOG_LEVEL`

gc.go

@@ -8,8 +8,13 @@ import (
 )
 
 // GC is a really stupid simple algorithm where we just delete things until
-// weve deleted enough things
+// we've deleted enough things. It is no-op if the current setup does not have
+// a (local) blockstore.
 func (nd *Node) GC(ctx context.Context, todelete int64) error {
+	if nd.blockstore == nil {
+		return nil
+	}
+
 	keys, err := nd.blockstore.AllKeysChan(ctx)
 	if err != nil {
 		return err

gc_test.go

@@ -12,7 +12,9 @@ import (
 func TestPeriodicGC(t *testing.T) {
 	t.Parallel()
 
-	gnd := mustTestNode(t, Config{})
+	gnd := mustTestNode(t, Config{
+		Bitswap: true,
+	})
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

handler_test.go

@@ -12,6 +12,7 @@ func TestTrustless(t *testing.T) {
 	t.Parallel()
 
 	ts, gnd := mustTestServer(t, Config{
+		Bitswap:                 true,
 		TrustlessGatewayDomains: []string{"trustless.com"},
 	})
 

handlers.go

@@ -85,12 +85,27 @@ func withRequestLogger(next http.Handler) http.Handler {
 }
 
 func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) {
-	backend, err := gateway.NewBlocksBackend(
-		nd.bsrv,
+	var (
+		backend gateway.IPFSBackend
+		err     error
+	)
+
+	options := []gateway.BackendOption{
 		gateway.WithValueStore(nd.vs),
 		gateway.WithNameSystem(nd.ns),
-		gateway.WithResolver(nd.resolver),
-	)
+		gateway.WithResolver(nd.resolver), // May be nil, but that is fine.
+	}
+
+	if len(cfg.RemoteBackends) > 0 && cfg.RemoteBackendMode == RemoteBackendCAR {
+		var fetcher gateway.CarFetcher
+		fetcher, err = gateway.NewRemoteCarFetcher(cfg.RemoteBackends, nil)
+		if err != nil {
+			return nil, err
+		}
+		backend, err = gateway.NewCarBackend(fetcher, options...)
+	} else {
+		backend, err = gateway.NewBlocksBackend(nd.bsrv, options...)
+	}
 	if err != nil {
 		return nil, err
 	}