diff --git a/.release-please-manifest.json b/.release-please-manifest.json index fe2a5bbc..3bab71b9 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -13,5 +13,6 @@ "packages/unixfs": "3.0.0", "packages/utils": "0.0.1", "packages/http": "1.0.1", - "packages/routers": "1.0.0" + "packages/routers": "1.0.0", + "packages/verified-fetch": "0.0.0" } diff --git a/.release-please.json b/.release-please.json index 10d16aa8..f38bc1fb 100644 --- a/.release-please.json +++ b/.release-please.json @@ -23,6 +23,7 @@ "packages/routers": {}, "packages/strings": {}, "packages/unixfs": {}, - "packages/utils": {} + "packages/utils": {}, + "packages/verified-fetch": {} } } diff --git a/packages/interop/package.json b/packages/interop/package.json index 4e116af1..202daae3 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -70,9 +70,10 @@ "@helia/routers": "^1.0.0", "@helia/strings": "^3.0.0", "@helia/unixfs": "^3.0.0", + "@helia/verified-fetch": "^0.0.0", "@ipld/car": "^5.2.5", - "@ipld/dag-cbor": "^9.0.7", - "@libp2p/interface": "^1.1.1", + "@ipld/dag-cbor": "^9.1.0", + "@libp2p/interface": "^1.1.2", "@libp2p/kad-dht": "^12.0.2", "@libp2p/keychain": "^4.0.7", "@libp2p/peer-id": "^4.0.5", diff --git a/packages/interop/src/fixtures/data/QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car b/packages/interop/src/fixtures/data/QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car new file mode 100644 index 00000000..a1279473 Binary files /dev/null and b/packages/interop/src/fixtures/data/QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car differ diff --git a/packages/interop/src/fixtures/data/QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR-xkcd-Barrel-part-1.car b/packages/interop/src/fixtures/data/QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR-xkcd-Barrel-part-1.car new file mode 100644 index 00000000..362d3854 Binary files /dev/null and b/packages/interop/src/fixtures/data/QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR-xkcd-Barrel-part-1.car differ diff --git a/packages/interop/src/fixtures/data/QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car b/packages/interop/src/fixtures/data/QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car new file mode 100644 index 00000000..5391b246 Binary files /dev/null and b/packages/interop/src/fixtures/data/QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car differ diff --git a/packages/interop/src/fixtures/data/QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car b/packages/interop/src/fixtures/data/QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car new file mode 100644 index 00000000..de186e8f Binary files /dev/null and b/packages/interop/src/fixtures/data/QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car differ diff --git a/packages/interop/src/fixtures/data/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car b/packages/interop/src/fixtures/data/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car new file mode 100644 index 00000000..bc2ae755 Binary files /dev/null and b/packages/interop/src/fixtures/data/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car differ diff --git a/packages/interop/src/fixtures/load-fixture-data.ts b/packages/interop/src/fixtures/load-fixture-data.ts new file mode 100644 index 00000000..12c14c4b --- /dev/null +++ b/packages/interop/src/fixtures/load-fixture-data.ts @@ -0,0 +1,9 @@ +import loadFixture from 'aegir/fixtures' +import drain from 'it-drain' +import type { Controller } from 'ipfsd-ctl' + +export async function loadFixtureDataCar (controller: Controller, path: string): Promise { + const fixtureData = `src/fixtures/data/${path}` + const buf = loadFixture(fixtureData) + await drain(controller.api.dag.import([buf])) +} diff --git a/packages/interop/src/verified-fetch-json.spec.ts b/packages/interop/src/verified-fetch-json.spec.ts new file mode 100644 index 00000000..62e6896d --- /dev/null +++ b/packages/interop/src/verified-fetch-json.spec.ts @@ -0,0 +1,48 @@ +/* eslint-env mocha */ +import { createVerifiedFetch } from '@helia/verified-fetch' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { createKuboNode } from './fixtures/create-kubo.js' +import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' +import type { Controller } from 'ipfsd-ctl' + +describe('@helia/verified-fetch - json', () => { + describe('unixfs - multiblock', () => { + let controller: Controller<'go'> + let verifiedFetch: Awaited> + + before(async () => { + controller = await createKuboNode() + await controller.start() + // As of 2024-01-18, https://cloudflare-ipfs.com/ipns/tokens.uniswap.org resolves to: + // root: QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr + // child1: QmNik5N4ryNwzzXYq5hCYKGcRjAf9QtigxtiJh9o8aXXbG // partial JSON + // child2: QmWNBJX6fZyNTLWNYBHxAHpBctCP43R2zeqV2G8uavqFZn // partial JSON + await loadFixtureDataCar(controller, 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car') + verifiedFetch = await createVerifiedFetch({ + gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], + // Temporarily disabling delegated routers in browser until CORS issue is fixed. see https://github.com/ipshipyard/waterworks-community/issues/4 + routers: process.env.RUNNER_ENV === 'node' ? [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] : [] + }) + }) + + after(async () => { + await controller.stop() + await verifiedFetch.stop() + }) + + it('handles UnixFS-chunked JSON file', async () => { + const resp = await verifiedFetch(CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')) + expect(resp).to.be.ok() + const jsonObj = await resp.json() + expect(jsonObj).to.be.ok() + expect(jsonObj).to.have.property('name').equal('Uniswap Labs Default') + expect(jsonObj).to.have.property('timestamp').equal('2023-12-13T18:25:25.830Z') + expect(jsonObj).to.have.property('version').to.deep.equal({ major: 11, minor: 11, patch: 0 }) + expect(jsonObj).to.have.property('tags') + expect(jsonObj).to.have.property('logoURI').equal('ipfs://QmNa8mQkrNKp1WEEeGjFezDmDeodkWRevGFN8JCV7b4Xir') + expect(jsonObj).to.have.property('keywords').to.deep.equal(['uniswap', 'default']) + expect(jsonObj.tokens).to.be.an('array').of.length(767) + }) + }) +}) diff --git a/packages/interop/src/verified-fetch-unixfs-dir.spec.ts b/packages/interop/src/verified-fetch-unixfs-dir.spec.ts new file mode 100644 index 00000000..415b04a2 --- /dev/null +++ b/packages/interop/src/verified-fetch-unixfs-dir.spec.ts @@ -0,0 +1,76 @@ +/* eslint-env mocha */ +import { createVerifiedFetch } from '@helia/verified-fetch' +import { expect } from 'aegir/chai' +import { createKuboNode } from './fixtures/create-kubo.js' +import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' +import type { Controller } from 'ipfsd-ctl' + +describe('@helia/verified-fetch - unixfs directory', () => { + let controller: Controller + let verifiedFetch: Awaited> + + before(async () => { + controller = await createKuboNode() + await controller.start() + verifiedFetch = await createVerifiedFetch({ + gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], + // Temporarily disabling delegated routers in browser until CORS issue is fixed. see https://github.com/ipshipyard/waterworks-community/issues/4 + routers: process.env.RUNNER_ENV === 'node' ? [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] : [] + }) + }) + + after(async () => { + await controller.stop() + await verifiedFetch.stop() + }) + + describe('XKCD Barrel Part 1', () => { + before(async () => { + // This is the content of https://explore.ipld.io/#/explore/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1%20-%20Barrel%20-%20Part%201 + await loadFixtureDataCar(controller, 'QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR-xkcd-Barrel-part-1.car') + }) + + it('fails to load when passed the root', async () => { + // The spec says we should generate HTML with directory listings, but we don't do that yet, so expect a failure + const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR') + expect(resp).to.be.ok() + expect(resp.status).to.equal(501) // TODO: we should do a directory listing instead + }) + + it('can return a string for unixfs pathed data', async () => { + const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt') + expect(resp).to.be.ok() + const text = await resp.text() + expect(text).to.equal('Don\'t we all.') + expect(resp.headers.get('content-type')).to.equal('text/plain') + }) + + it('can return an image for unixfs pathed data', async () => { + const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png') + expect(resp).to.be.ok() + expect(resp.headers.get('content-type')).to.equal('image/png') + const imgData = await resp.blob() + expect(imgData).to.be.ok() + expect(imgData.size).to.equal(24848) + }) + }) + + // TODO: find a smaller car file so the test doesn't timeout locally or flake on CI + describe.skip('HAMT-sharded directory', () => { + before(async () => { + // from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car + await loadFixtureDataCar(controller, 'bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car') + }) + + it('loads path /ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', async () => { + const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt') + expect(resp).to.be.ok() + expect(resp.headers.get('content-type')).to.equal('text/plain') + const text = await resp.text() + // npx kubo@0.25.0 cat '/ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt' + expect(text).to.equal(`Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc non imperdiet nunc. Proin ac quam ut nibh eleifend aliquet. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Sed ligula dolor, imperdiet sagittis arcu et, semper tincidunt urna. Donec et tempor augue, quis sollicitudin metus. Curabitur semper ullamcorper aliquet. Mauris hendrerit sodales lectus eget fermentum. Proin sollicitudin vestibulum commodo. Vivamus nec lectus eu augue aliquet dignissim nec condimentum justo. In hac habitasse platea dictumst. Mauris vel sem neque. + +Vivamus finibus, enim at lacinia semper, arcu erat gravida lacus, sit amet gravida magna orci sit amet est. Sed non leo lacus. Nullam viverra ipsum a tincidunt dapibus. Nulla pulvinar ligula sit amet ante ultrices tempus. Proin purus urna, semper sed lobortis quis, gravida vitae ipsum. Aliquam mi urna, pulvinar eu bibendum quis, convallis ac dolor. In gravida justo sed risus ullamcorper, vitae luctus massa hendrerit. Pellentesque habitant amet.`) + }) + }) +}) diff --git a/packages/interop/src/verified-fetch-websites.spec.ts b/packages/interop/src/verified-fetch-websites.spec.ts new file mode 100644 index 00000000..c6662164 --- /dev/null +++ b/packages/interop/src/verified-fetch-websites.spec.ts @@ -0,0 +1,87 @@ +/* eslint-env mocha */ +import { createVerifiedFetch } from '@helia/verified-fetch' +import { expect } from 'aegir/chai' +import { createKuboNode } from './fixtures/create-kubo.js' +import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' +import type { Controller } from 'ipfsd-ctl' + +describe('@helia/verified-fetch - websites', () => { + describe('helia-identify.on.fleek.co', () => { + let controller: Controller<'go'> + let verifiedFetch: Awaited> + + before(async () => { + controller = await createKuboNode() + await controller.start() + // 2024-01-22 CID for _dnslink.helia-identify.on.fleek.co + await loadFixtureDataCar(controller, 'QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car') + verifiedFetch = await createVerifiedFetch({ + gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], + // Temporarily disabling delegated routers in browser until CORS issue is fixed. see https://github.com/ipshipyard/waterworks-community/issues/4 + routers: process.env.RUNNER_ENV === 'node' ? [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] : [] + }) + }) + + after(async () => { + await controller.stop() + await verifiedFetch.stop() + }) + + it('loads index.html when passed helia-identify.on.fleek.co root CID', async () => { + const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv') + expect(resp).to.be.ok() + const html = await resp.text() + expect(html).to.be.ok() + expect(html).to.include('Run Identify on a remote node with Helia') + }) + + it('loads helia-identify.on.fleek.co index.html directly ', async () => { + const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv/index.html') + expect(resp).to.be.ok() + const html = await resp.text() + expect(html).to.be.ok() + expect(html).to.include('Run Identify on a remote node with Helia') + }) + }) + + /** + * + * Created on 2024-01-23. /ipns/blog.libp2p.io/index.html resolved to QmVZNGy6SPvUbvQCXXaGDdp8kvfJm9MMozjU12dyzH6hKf + * + * ```shell + * mkdir fake-blog.libp2p.io + * npx kubo@0.25.0 cat '/ipfs/QmVZNGy6SPvUbvQCXXaGDdp8kvfJm9MMozjU12dyzH6hKf' > fake-blog.libp2p.io/index.html + * npx kubo@0.25.0 add -r fake-blog.libp2p.io + * npx kubo@0.25.0 dag export QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw > QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car + * ``` + */ + describe('fake blog.libp2p.io', () => { + let controller: Controller<'go'> + let verifiedFetch: Awaited> + + before(async () => { + controller = await createKuboNode() + await controller.start() + await loadFixtureDataCar(controller, 'QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car') + verifiedFetch = await createVerifiedFetch({ + gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], + // Temporarily disabling delegated routers in browser until CORS issue is fixed. see https://github.com/ipshipyard/waterworks-community/issues/4 + routers: process.env.RUNNER_ENV === 'node' ? [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] : [] + }) + }) + + after(async () => { + await controller.stop() + await verifiedFetch.stop() + }) + + it('loads index.html when passed fake-blog.libp2p.io root CID', async () => { + const resp = await verifiedFetch('ipfs://QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw') + expect(resp).to.be.ok() + const html = await resp.text() + expect(html).to.be.ok() + expect(html).to.include('Home | libp2p Blog & News') + expect(html).to.include('') + }) + }) +}) diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json index bc845a1b..79ff2ae8 100644 --- a/packages/interop/tsconfig.json +++ b/packages/interop/tsconfig.json @@ -46,6 +46,9 @@ }, { "path": "../unixfs" + }, + { + "path": "../verified-fetch" } ] } diff --git a/packages/verified-fetch/LICENSE b/packages/verified-fetch/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/verified-fetch/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/verified-fetch/LICENSE-APACHE b/packages/verified-fetch/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/verified-fetch/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/verified-fetch/LICENSE-MIT b/packages/verified-fetch/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/verified-fetch/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md new file mode 100644 index 00000000..60271681 --- /dev/null +++ b/packages/verified-fetch/README.md @@ -0,0 +1,262 @@ +

+ + Helia logo + +

+ +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain) + +> A fetch-like API for obtaining verified & trustless IPFS content on the web. + +# About + +`@helia/verified-fetch` is a library that provides a fetch-like API for fetching trustless content from IPFS and verifying it. + +This library should act as a replacement for the `fetch()` API for fetching content from IPFS, and will return a [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response) object that can be used in a similar manner to the `fetch()` API. This means browser and HTTP caching inside browser main threads, web-workers, and service workers, as well as other features of the `fetch()` API should work in a way familiar to developers. + +Exports a `createVerifiedFetch` function that returns a `fetch()` like API method Helia for fetching IPFS content. + +You may use any supported resource argument to fetch content: + +- CID instance +- IPFS URL +- IPNS URL + +## Example + +```typescript +import { createVerifiedFetch } from '@helia/verified-fetch' + +const fetch = await createVerifiedFetch({ + gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] +}) + +const resp = await fetch('ipfs://bafy...') + +const json = await resp.json() +``` + +## Example - Using a CID instance to fetch JSON + +```typescript +import { createVerifiedFetch } from '@helia/verified-fetch' +import { CID } from 'multiformats/cid' + +const fetch = await createVerifiedFetch({ + gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] +}) + +const cid = CID.parse('bafyFoo') // some image file +const response = await fetch(cid) +const json = await response.json() +``` + +## Example - Using IPFS protocol to fetch an image + +```typescript +import { createVerifiedFetch } from '@helia/verified-fetch' + +const fetch = await createVerifiedFetch({ + gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] +}) +const response = await fetch('ipfs://bafyFoo') // CID for some image file +const blob = await response.blob() +const image = document.createElement('img') +image.src = URL.createObjectURL(blob) +document.body.appendChild(image) +``` + +## Example - Using IPNS protocol to stream a big file + +```typescript +import { createVerifiedFetch } from '@helia/verified-fetch' + +const fetch = await createVerifiedFetch({ + gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] +}) +const response = await fetch('ipns://mydomain.com/path/to/very-long-file.log') +const bigFileStreamReader = await response.body.getReader() +``` + +### Configuration + +#### Usage with customized Helia + +You can see variations of Helia and js-libp2p configuration options at . + +The `@helia/http` module is currently in-progress, but the init options should be a subset of the `helia` module's init options. See for more information. + +```typescript +import { trustlessGateway } from '@helia/block-brokers' +import { createHeliaHTTP } from '@helia/http' +import { delegatedHTTPRouting } from '@helia/routers' +import { createVerifiedFetch } from '@helia/verified-fetch' + +const fetch = await createVerifiedFetch( + await createHeliaHTTP({ + blockBrokers: [ + trustlessGateway({ + gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] + }) + ], + routers: ['http://delegated-ipfs.dev'].map((routerUrl) => delegatedHTTPRouting(routerUrl)) + }) +) + +const resp = await fetch('ipfs://bafy...') + +const json = await resp.json() +``` + +### Comparison to fetch + +First, this library will require instantiation in order to configure the gateways and delegated routers, or potentially a custom Helia instance. Secondly, once your verified-fetch method is created, it will act as similar to the `fetch()` API as possible. + +[The `fetch()` API](https://developer.mozilla.org/en-US/docs/Web/API/fetch) takes two parameters: + +1. A [resource](https://developer.mozilla.org/en-US/docs/Web/API/fetch#resource) +2. An [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) + +#### Resource argument + +This library intends to support the following methods of fetching web3 content from IPFS: + +1. IPFS protocol: `ipfs://` & `ipfs://` +2. IPNS protocol: `ipns://` & `ipns://` & `ipns://` +3. CID instances: An actual CID instance `CID.parse('bafy...')` + +As well as support for pathing & params for item 1&2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453). + +If you pass a CID instance, we assume you want the content for that specific CID only, and do not support pathing or params for that CID. + +#### Options argument + +This library does not plan to support the exact Fetch API options object, as some of the arguments don't make sense. Instead, it will only support options necessary to meet [IPFS specs](https://specs.ipfs.tech/) related to specifying the resultant shape of desired content. + +Some of those header specifications are: + +1. +2. +3. + +Where possible, options and Helia internals will be automatically configured to the appropriate codec & content type based on the `verified-fetch` configuration and `options` argument passed. + +Known Fetch API options that will be supported: + +1. `signal` - An AbortSignal that a user can use to abort the request. +2. `redirect` - A string that specifies the redirect type. One of `follow`, `error`, or `manual`. Defaults to `follow`. Best effort to adhere to the [Fetch API redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect) parameter. +3. `headers` - An object of headers to be sent with the request. Best effort to adhere to the [Fetch API headers](https://developer.mozilla.org/en-US/docs/Web/API/fetch#headers) parameter. + - `accept` - A string that specifies the accept header. Relevant values: + - [`vnd.ipld.raw`](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw). (default) + - [`vnd.ipld.car`](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) + - [`vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record) +4. `method` - A string that specifies the HTTP method to use for the request. Defaults to `GET`. Best effort to adhere to the [Fetch API method](https://developer.mozilla.org/en-US/docs/Web/API/fetch#method) parameter. +5. `body` - An object that specifies the body of the request. Best effort to adhere to the [Fetch API body](https://developer.mozilla.org/en-US/docs/Web/API/fetch#body) parameter. +6. `cache` - Will basically act as `force-cache` for the request. Best effort to adhere to the [Fetch API cache](https://developer.mozilla.org/en-US/docs/Web/API/fetch#cache) parameter. + +Non-Fetch API options that will be supported: + +1. `onProgress` - Similar to Helia `onProgress` options, this will be a function that will be called with a progress event. Supported progress events are: + - `helia:verified-fetch:error` - An error occurred during the request. + - `helia:verified-fetch:request:start` - The request has been sent + - `helia:verified-fetch:request:complete` - The request has been sent + - `helia:verified-fetch:request:error` - An error occurred during the request. + - `helia:verified-fetch:request:abort` - The request was aborted prior to completion. + - `helia:verified-fetch:response:start` - The initial HTTP Response headers have been set, and response stream is started. + - `helia:verified-fetch:response:complete` - The response stream has completed. + - `helia:verified-fetch:response:error` - An error occurred while building the response. + +Some in-flight specs (IPIPs) that will affect the options object this library supports in the future can be seen at , a subset are: + +1. [IPIP-0412: Signaling Block Order in CARs on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0412/) +2. [IPIP-0402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/) +3. [IPIP-0386: Subdomain Gateway Interop with \_redirects](https://specs.ipfs.tech/ipips/ipip-0386/) +4. [IPIP-0328: JSON and CBOR Response Formats on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0328/) +5. [IPIP-0288: TAR Response Format on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0288/) + +#### Response types + +This library's purpose is to return reasonably representable content from IPFS. In other words, fetching content is intended for leaf-node content -- such as images/videos/audio & other assets, or other IPLD content (with link) -- that can be represented by . The content type you receive back will depend upon the CID you request as well as the `Accept` header value you provide. + +All content we retrieve from the IPFS network is obtained via an AsyncIterable, and will be set as the [body of the HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response/Response#body) via a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream) or other efficient method that avoids loading the entire response into memory or getting the entire response from the network before returning a response to the user. + +If your content doesn't have a mime-type or an [IPFS spec](https://specs.ipfs.tech), this library will not support it, but you can use the [`helia`](https://github.com/ipfs/helia) library directly for those use cases. See [Unsupported response types](#unsupported-response-types) for more information. + +##### Handling response types + +For handling responses we want to follow conventions/abstractions from Fetch API where possible: + +- For JSON, assuming you abstract any differences between dag-json/dag-cbor/json/and json-file-on-unixfs, you would call `.json()` to get a JSON object. +- For images (or other web-relevant asset) you want to add to the DOM, use `.blob()` or `.arrayBuffer()` to get the raw bytes. +- For plain text in utf-8, you would call `.text()` +- For streaming response data, use something like `response.body.getReader()` to get a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream). + +##### Unsupported response types + +- Returning IPLD nodes or DAGs as JS objects is not supported, as there is no currently well-defined structure for representing this data in an [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response). Instead, users should request `aplication/vnd.ipld.car` or use the [`helia`](https://github.com/ipfs/helia) library directly for this use case. +- Others? Open an issue or PR! + +#### Response headers + +This library will set the [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) headers to the appropriate values for the content type according to the appropriate [IPFS Specifications](https://specs.ipfs.tech/). + +Some known header specifications: + +- +- +- + +#### Possible Scenarios that could cause confusion + +##### Attempting to fetch the CID for content that does not make sense + +If you request `bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze`, which points to the root of the en.wikipedia.org mirror, a response object does not make sense. + +#### Errors + +Known Errors that can be thrown: + +1. `TypeError` - If the resource argument is not a string, CID, or CID string. +2. `TypeError` - If the options argument is passed and not an object. +3. `TypeError` - If the options argument is passed and is malformed. +4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal. + +# Install + +```console +$ npm i @helia/verified-fetch +``` + +## Browser ` +``` + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json new file mode 100644 index 00000000..007cb8d9 --- /dev/null +++ b/packages/verified-fetch/package.json @@ -0,0 +1,176 @@ +{ + "name": "@helia/verified-fetch", + "version": "0.0.0", + "description": "A fetch-like API for obtaining verified & trustless IPFS content on the web.", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia/tree/main/packages/verified-fetch#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia/issues" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "keywords": [ + "IPFS", + "fetch", + "helia" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "project": true, + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/block-brokers": "^2.0.1", + "@helia/dag-cbor": "^3.0.0", + "@helia/dag-json": "^3.0.0", + "@helia/http": "^1.0.1", + "@helia/interface": "^4.0.0", + "@helia/ipns": "^6.0.0", + "@helia/json": "^3.0.0", + "@helia/routers": "^1.0.0", + "@helia/unixfs": "^3.0.0", + "@ipld/dag-cbor": "^9.1.0", + "@ipld/dag-json": "^10.1.7", + "@ipld/dag-pb": "^4.0.8", + "@libp2p/interface": "^1.1.2", + "@libp2p/peer-id": "^4.0.5", + "hashlru": "^2.3.0", + "ipfs-unixfs-exporter": "^13.4.0", + "mime-types": "^2.1.35", + "multiformats": "^13.0.0", + "progress-events": "^1.0.0" + }, + "devDependencies": { + "@libp2p/logger": "^4.0.5", + "@libp2p/peer-id-factory": "^4.0.5", + "@types/mime-types": "^2.1.4", + "@types/sinon": "^17.0.2", + "aegir": "^42.1.0", + "helia": "^4.0.1", + "interface-blockstore": "^5.2.9", + "sinon": "^17.0.1", + "sinon-ts": "^2.0.0" + }, + "sideEffects": false +} diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts new file mode 100644 index 00000000..9aa4e17e --- /dev/null +++ b/packages/verified-fetch/src/index.ts @@ -0,0 +1,310 @@ +/** + * @packageDocumentation + * + * `@helia/verified-fetch` is a library that provides a fetch-like API for fetching trustless content from IPFS and verifying it. + * + * This library should act as a replacement for the `fetch()` API for fetching content from IPFS, and will return a [`Response`](https://developer.mozilla.org/en-US/docs/Web/API/Response) object that can be used in a similar manner to the `fetch()` API. This means browser and HTTP caching inside browser main threads, web-workers, and service workers, as well as other features of the `fetch()` API should work in a way familiar to developers. + * + * Exports a `createVerifiedFetch` function that returns a `fetch()` like API method {@link Helia} for fetching IPFS content. + * + * You may use any supported resource argument to fetch content: + * + * - CID instance + * - IPFS URL + * - IPNS URL + * + * @example + * + * ```typescript + * import { createVerifiedFetch } from '@helia/verified-fetch' + * + * const fetch = await createVerifiedFetch({ + * gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] + *}) + * + * const resp = await fetch('ipfs://bafy...') + * + * const json = await resp.json() + *``` + * + * + * @example Using a CID instance to fetch JSON + * + * ```typescript + * import { createVerifiedFetch } from '@helia/verified-fetch' + * import { CID } from 'multiformats/cid' + * + * const fetch = await createVerifiedFetch({ + * gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] + * }) + * + * const cid = CID.parse('bafyFoo') // some image file + * const response = await fetch(cid) + * const json = await response.json() + * ``` + * + * @example Using IPFS protocol to fetch an image + * + * ```typescript + * import { createVerifiedFetch } from '@helia/verified-fetch' + * + * const fetch = await createVerifiedFetch({ + * gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] + * }) + * const response = await fetch('ipfs://bafyFoo') // CID for some image file + * const blob = await response.blob() + * const image = document.createElement('img') + * image.src = URL.createObjectURL(blob) + * document.body.appendChild(image) + * ``` + * + * @example Using IPNS protocol to stream a big file + * + * ```typescript + * import { createVerifiedFetch } from '@helia/verified-fetch' + * + * const fetch = await createVerifiedFetch({ + * gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] + * }) + * const response = await fetch('ipns://mydomain.com/path/to/very-long-file.log') + * const bigFileStreamReader = await response.body.getReader() + * ``` + * + * ### Configuration + * + * #### Usage with customized Helia + * + * You can see variations of Helia and js-libp2p configuration options at https://helia.io/interfaces/helia.index.HeliaInit.html. + * + * The `@helia/http` module is currently in-progress, but the init options should be a subset of the `helia` module's init options. See https://github.com/ipfs/helia/issues/289 for more information. + * + * ```typescript + * import { trustlessGateway } from '@helia/block-brokers' + * import { createHeliaHTTP } from '@helia/http' + * import { delegatedHTTPRouting } from '@helia/routers' + * import { createVerifiedFetch } from '@helia/verified-fetch' + * + * const fetch = await createVerifiedFetch( + * await createHeliaHTTP({ + * blockBrokers: [ + * trustlessGateway({ + * gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] + * }) + * ], + * routers: ['http://delegated-ipfs.dev'].map((routerUrl) => delegatedHTTPRouting(routerUrl)) + * }) + * ) + * + * const resp = await fetch('ipfs://bafy...') + * + * const json = await resp.json() + * ``` + * + * ### Comparison to fetch + * + * First, this library will require instantiation in order to configure the gateways and delegated routers, or potentially a custom Helia instance. Secondly, once your verified-fetch method is created, it will act as similar to the `fetch()` API as possible. + * + * [The `fetch()` API](https://developer.mozilla.org/en-US/docs/Web/API/fetch) takes two parameters: + * + * 1. A [resource](https://developer.mozilla.org/en-US/docs/Web/API/fetch#resource) + * 2. An [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) + * + * #### Resource argument + * + * This library intends to support the following methods of fetching web3 content from IPFS: + * + * 1. IPFS protocol: `ipfs://` & `ipfs://` + * 2. IPNS protocol: `ipns://` & `ipns://` & `ipns://` + * 3. CID instances: An actual CID instance `CID.parse('bafy...')` + * + * As well as support for pathing & params for item 1&2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453). + * + * If you pass a CID instance, we assume you want the content for that specific CID only, and do not support pathing or params for that CID. + * + * #### Options argument + * + * This library does not plan to support the exact Fetch API options object, as some of the arguments don't make sense. Instead, it will only support options necessary to meet [IPFS specs](https://specs.ipfs.tech/) related to specifying the resultant shape of desired content. + * + * Some of those header specifications are: + * + * 1. https://specs.ipfs.tech/http-gateways/path-gateway/#request-headers + * 2. https://specs.ipfs.tech/http-gateways/trustless-gateway/#request-headers + * 3. https://specs.ipfs.tech/http-gateways/subdomain-gateway/#request-headers + * + * Where possible, options and Helia internals will be automatically configured to the appropriate codec & content type based on the `verified-fetch` configuration and `options` argument passed. + * + * Known Fetch API options that will be supported: + * + * 1. `signal` - An AbortSignal that a user can use to abort the request. + * 2. `redirect` - A string that specifies the redirect type. One of `follow`, `error`, or `manual`. Defaults to `follow`. Best effort to adhere to the [Fetch API redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect) parameter. + * 3. `headers` - An object of headers to be sent with the request. Best effort to adhere to the [Fetch API headers](https://developer.mozilla.org/en-US/docs/Web/API/fetch#headers) parameter. + * - `accept` - A string that specifies the accept header. Relevant values: + * - [`vnd.ipld.raw`](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw). (default) + * - [`vnd.ipld.car`](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) + * - [`vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record) + * 4. `method` - A string that specifies the HTTP method to use for the request. Defaults to `GET`. Best effort to adhere to the [Fetch API method](https://developer.mozilla.org/en-US/docs/Web/API/fetch#method) parameter. + * 5. `body` - An object that specifies the body of the request. Best effort to adhere to the [Fetch API body](https://developer.mozilla.org/en-US/docs/Web/API/fetch#body) parameter. + * 6. `cache` - Will basically act as `force-cache` for the request. Best effort to adhere to the [Fetch API cache](https://developer.mozilla.org/en-US/docs/Web/API/fetch#cache) parameter. + * + * + * Non-Fetch API options that will be supported: + * + * 1. `onProgress` - Similar to Helia `onProgress` options, this will be a function that will be called with a progress event. Supported progress events are: + * - `helia:verified-fetch:error` - An error occurred during the request. + * - `helia:verified-fetch:request:start` - The request has been sent + * - `helia:verified-fetch:request:complete` - The request has been sent + * - `helia:verified-fetch:request:error` - An error occurred during the request. + * - `helia:verified-fetch:request:abort` - The request was aborted prior to completion. + * - `helia:verified-fetch:response:start` - The initial HTTP Response headers have been set, and response stream is started. + * - `helia:verified-fetch:response:complete` - The response stream has completed. + * - `helia:verified-fetch:response:error` - An error occurred while building the response. + * + * Some in-flight specs (IPIPs) that will affect the options object this library supports in the future can be seen at https://specs.ipfs.tech/ipips, a subset are: + * + * 1. [IPIP-0412: Signaling Block Order in CARs on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0412/) + * 2. [IPIP-0402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/) + * 3. [IPIP-0386: Subdomain Gateway Interop with _redirects](https://specs.ipfs.tech/ipips/ipip-0386/) + * 4. [IPIP-0328: JSON and CBOR Response Formats on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0328/) + * 5. [IPIP-0288: TAR Response Format on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0288/) + * + * #### Response types + * + * This library's purpose is to return reasonably representable content from IPFS. In other words, fetching content is intended for leaf-node content -- such as images/videos/audio & other assets, or other IPLD content (with link) -- that can be represented by https://developer.mozilla.org/en-US/docs/Web/API/Response#instance_methods. The content type you receive back will depend upon the CID you request as well as the `Accept` header value you provide. + * + * All content we retrieve from the IPFS network is obtained via an AsyncIterable, and will be set as the [body of the HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response/Response#body) via a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream) or other efficient method that avoids loading the entire response into memory or getting the entire response from the network before returning a response to the user. + * + * If your content doesn't have a mime-type or an [IPFS spec](https://specs.ipfs.tech), this library will not support it, but you can use the [`helia`](https://github.com/ipfs/helia) library directly for those use cases. See [Unsupported response types](#unsupported-response-types) for more information. + * + * ##### Handling response types + * + * For handling responses we want to follow conventions/abstractions from Fetch API where possible: + * + * - For JSON, assuming you abstract any differences between dag-json/dag-cbor/json/and json-file-on-unixfs, you would call `.json()` to get a JSON object. + * - For images (or other web-relevant asset) you want to add to the DOM, use `.blob()` or `.arrayBuffer()` to get the raw bytes. + * - For plain text in utf-8, you would call `.text()` + * - For streaming response data, use something like `response.body.getReader()` to get a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream). + * + * ##### Unsupported response types + * + * * Returning IPLD nodes or DAGs as JS objects is not supported, as there is no currently well-defined structure for representing this data in an [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response). Instead, users should request `aplication/vnd.ipld.car` or use the [`helia`](https://github.com/ipfs/helia) library directly for this use case. + * * Others? Open an issue or PR! + * + * #### Response headers + * + * This library will set the [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) headers to the appropriate values for the content type according to the appropriate [IPFS Specifications](https://specs.ipfs.tech/). + * + * Some known header specifications: + * + * * https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers + * * https://specs.ipfs.tech/http-gateways/trustless-gateway/#response-headers + * * https://specs.ipfs.tech/http-gateways/subdomain-gateway/#response-headers + * + * #### Possible Scenarios that could cause confusion + * + * ##### Attempting to fetch the CID for content that does not make sense + * + * If you request `bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze`, which points to the root of the en.wikipedia.org mirror, a response object does not make sense. + * + * #### Errors + * + * Known Errors that can be thrown: + * + * 1. `TypeError` - If the resource argument is not a string, CID, or CID string. + * 2. `TypeError` - If the options argument is passed and not an object. + * 3. `TypeError` - If the options argument is passed and is malformed. + * 4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal. + */ + +import { trustlessGateway } from '@helia/block-brokers' +import { createHeliaHTTP } from '@helia/http' +import { delegatedHTTPRouting } from '@helia/routers' +import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' +import type { Helia } from '@helia/interface' +import type { IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' +import type { GetEvents } from '@helia/unixfs' +import type { CID } from 'multiformats/cid' +import type { ProgressEvent, ProgressOptions } from 'progress-events' + +/** + * The types for the first argument of the `verifiedFetch` function. + */ +export type Resource = string | CID + +export interface CIDDetail { + cid: string + path: string +} + +export interface CIDDetailError extends CIDDetail { + error: Error +} + +export interface VerifiedFetch { + (resource: Resource, options?: VerifiedFetchInit): Promise + start(): Promise + stop(): Promise +} + +/** + * Instead of passing a Helia instance, you can pass a list of gateways and routers, and a HeliaHTTP instance will be created for you. + */ +export interface CreateVerifiedFetchWithOptions { + gateways: string[] + routers?: string[] +} + +export type BubbledProgressEvents = + // unixfs + GetEvents | + // ipns + ResolveProgressEvents | ResolveDnsLinkProgressEvents | IPNSRoutingEvents + +export type VerifiedFetchProgressEvents = + ProgressEvent<'verified-fetch:request:start', CIDDetail> | + ProgressEvent<'verified-fetch:request:info', string> | + ProgressEvent<'verified-fetch:request:progress:chunk', CIDDetail> | + ProgressEvent<'verified-fetch:request:end', CIDDetail> | + ProgressEvent<'verified-fetch:request:error', CIDDetailError> + +/** + * Options for the `fetch` function returned by `createVerifiedFetch`. + * + * This method accepts all the same options as the `fetch` function in the browser, plus an `onProgress` option to + * listen for progress events. + */ +export interface VerifiedFetchInit extends RequestInit, ProgressOptions { +} + +/** + * Create and return a Helia node + */ +export async function createVerifiedFetch (init: Helia | CreateVerifiedFetchWithOptions): Promise { + if (!isHelia(init)) { + init = await createHeliaHTTP({ + blockBrokers: [ + trustlessGateway({ + gateways: init.gateways + }) + ], + routers: init.routers?.map((routerUrl) => delegatedHTTPRouting(routerUrl)) + }) + } + + const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }) + async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise { + return verifiedFetchInstance.fetch(resource, options) + } + verifiedFetch.stop = verifiedFetchInstance.stop.bind(verifiedFetchInstance) + verifiedFetch.start = verifiedFetchInstance.start.bind(verifiedFetchInstance) + + return verifiedFetch +} + +function isHelia (obj: any): obj is Helia { + // test for the presence of known Helia properties, return a boolean value + return obj?.blockstore != null && + obj?.datastore != null && + obj?.gc != null && + obj?.stop != null && + obj?.start != null +} diff --git a/packages/verified-fetch/src/utils/get-content-type.ts b/packages/verified-fetch/src/utils/get-content-type.ts new file mode 100644 index 00000000..9552f9ed --- /dev/null +++ b/packages/verified-fetch/src/utils/get-content-type.ts @@ -0,0 +1,55 @@ +import mime from 'mime-types' + +interface TestInput { + bytes: Uint8Array + path: string +} + +type TestOutput = Promise + +export const DEFAULT_MIME_TYPE = 'application/octet-stream' + +const xmlRegex = /^(<\?xml[^>]+>)?[^<^\w]+ TestOutput> = [ + // svg + async ({ bytes }): TestOutput => xmlRegex.test(new TextDecoder().decode(bytes.slice(0, 64))) + ? 'image/svg+xml' + : undefined, + // testing file-type from path + async ({ path }): TestOutput => { + const mimeType = mime.lookup(path) + if (mimeType !== false) { + return mimeType + } + return undefined + } +] + +const overrides: Record = { + 'video/quicktime': 'video/mp4' +} + +/** + * Override the content type based on overrides. + */ +function overrideContentType (type: string): string { + return overrides[type] ?? type +} + +/** + * Get the content type from the input based on the tests. + */ +export async function getContentType (input: TestInput): Promise { + for (const test of tests) { + const type = await test(input) + if (type !== undefined) { + return overrideContentType(type) + } + } + return DEFAULT_MIME_TYPE +} diff --git a/packages/verified-fetch/src/utils/get-stream-and-content-type.ts b/packages/verified-fetch/src/utils/get-stream-and-content-type.ts new file mode 100644 index 00000000..09b45e11 --- /dev/null +++ b/packages/verified-fetch/src/utils/get-stream-and-content-type.ts @@ -0,0 +1,44 @@ +import { CustomProgressEvent } from 'progress-events' +import { getContentType } from './get-content-type.js' +import type { VerifiedFetchInit } from '../index.js' +import type { ComponentLogger } from '@libp2p/interface' + +/** + * Converts an async iterator of Uint8Array bytes to a stream and attempts to determine the content type of those bytes. + */ +export async function getStreamAndContentType (iterator: AsyncIterable, path: string, logger: ComponentLogger, options?: Pick): Promise<{ contentType: string, stream: ReadableStream }> { + const log = logger.forComponent('helia:verified-fetch:get-stream-and-content-type') + const reader = iterator[Symbol.asyncIterator]() + const { value, done } = await reader.next() + + if (done === true) { + log.error('No content found for path', path) + throw new Error('No content found') + } + + const contentType = await getContentType({ bytes: value, path }) + const stream = new ReadableStream({ + async start (controller) { + // the initial value is already available + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:progress:chunk')) + controller.enqueue(value) + }, + async pull (controller) { + const { value, done } = await reader.next() + + if (done === true) { + if (value != null) { + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:progress:chunk')) + controller.enqueue(value) + } + controller.close() + return + } + + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:progress:chunk')) + controller.enqueue(value) + } + }) + + return { contentType, stream } +} diff --git a/packages/verified-fetch/src/utils/parse-resource.ts b/packages/verified-fetch/src/utils/parse-resource.ts new file mode 100644 index 00000000..a20e82d4 --- /dev/null +++ b/packages/verified-fetch/src/utils/parse-resource.ts @@ -0,0 +1,40 @@ +import { CID } from 'multiformats/cid' +import { parseUrlString } from './parse-url-string.js' +import type { ParsedUrlStringResults } from './parse-url-string.js' +import type { Resource } from '../index.js' +import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' +import type { ComponentLogger } from '@libp2p/interface' +import type { ProgressOptions } from 'progress-events' + +export interface ParseResourceComponents { + ipns: IPNS + logger: ComponentLogger +} + +export interface ParseResourceOptions extends ProgressOptions { + +} +/** + * Handles the different use cases for the `resource` argument. + * The resource can represent an IPFS path, IPNS path, or CID. + * If the resource represents an IPNS path, we need to resolve it to a CID. + */ +export async function parseResource (resource: Resource, { ipns, logger }: ParseResourceComponents, options?: ParseResourceOptions): Promise { + if (typeof resource === 'string') { + return parseUrlString({ urlString: resource, ipns, logger }, { onProgress: options?.onProgress }) + } + + const cid = CID.asCID(resource) + + if (cid != null) { + // an actual CID + return { + cid, + protocol: 'ipfs', + path: '', + query: {} + } + } + + throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${resource}`) +} diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts new file mode 100644 index 00000000..5846c413 --- /dev/null +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -0,0 +1,139 @@ +import { peerIdFromString } from '@libp2p/peer-id' +import { CID } from 'multiformats/cid' +import { TLRU } from './tlru.js' +import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns' +import type { ComponentLogger } from '@libp2p/interface' +import type { ProgressOptions } from 'progress-events' + +const ipnsCache = new TLRU(1000) + +export interface ParseUrlStringInput { + urlString: string + ipns: IPNS + logger: ComponentLogger +} +export interface ParseUrlStringOptions extends ProgressOptions { + +} + +export interface ParsedUrlStringResults { + protocol: string + path: string + cid: CID + query: Record +} + +const URL_REGEX = /^(?ip[fn]s):\/\/(?[^/$?]+)\/?(?[^$?]*)\??(?.*)$/ + +/** + * A function that parses ipfs:// and ipns:// URLs, returning an object with easily recognizable properties. + * + * After determining the protocol successfully, we process the cidOrPeerIdOrDnsLink: + * * If it's ipfs, it parses the CID or throws an Aggregate error + * * If it's ipns, it attempts to resolve the PeerId and then the DNSLink. If both fail, an Aggregate error is thrown. + */ +export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStringInput, options?: ParseUrlStringOptions): Promise { + const log = logger.forComponent('helia:verified-fetch:parse-url-string') + const match = urlString.match(URL_REGEX) + + if (match == null || match.groups == null) { + throw new TypeError(`Invalid URL: ${urlString}, please use ipfs:// or ipns:// URLs only.`) + } + + const { protocol, cidOrPeerIdOrDnsLink, path: urlPath, queryString } = match.groups + + let cid: CID | undefined + let resolvedPath: string | undefined + const errors: Error[] = [] + + if (protocol === 'ipfs') { + try { + cid = CID.parse(cidOrPeerIdOrDnsLink) + } catch (err) { + log.error(err) + errors.push(new TypeError('Invalid CID for ipfs:// URL')) + } + } else { + let resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink) + + if (resolveResult != null) { + cid = resolveResult.cid + resolvedPath = resolveResult.path + log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid) + } else { + // protocol is ipns + log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink) + let peerId = null + + try { + peerId = peerIdFromString(cidOrPeerIdOrDnsLink) + resolveResult = await ipns.resolve(peerId, { onProgress: options?.onProgress }) + cid = resolveResult?.cid + resolvedPath = resolveResult?.path + log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid) + ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2) + } catch (err) { + if (peerId == null) { + log.error('Could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err) + errors.push(new TypeError(`Could not parse PeerId in ipns url "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`)) + } else { + log.error('Could not resolve PeerId %c', peerId, err) + errors.push(new TypeError(`Could not resolve PeerId "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`)) + } + } + + if (cid == null) { + log.trace('Attempting to resolve DNSLink for %s', cidOrPeerIdOrDnsLink) + + try { + resolveResult = await ipns.resolveDns(cidOrPeerIdOrDnsLink, { onProgress: options?.onProgress }) + cid = resolveResult?.cid + resolvedPath = resolveResult?.path + log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid) + ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2) + } catch (err) { + log.error('Could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err) + errors.push(err as Error) + } + } + } + } + + if (cid == null) { + throw new AggregateError(errors, `Invalid resource. Cannot determine CID from URL "${urlString}"`) + } + + // parse query string + const query: Record = {} + + if (queryString != null && queryString.length > 0) { + const queryParts = queryString.split('&') + for (const part of queryParts) { + const [key, value] = part.split('=') + query[key] = decodeURIComponent(value) + } + } + + /** + * join the path from resolve result & given path. + * e.g. /ipns// that is resolved to /ipfs//, when requested as /ipns//, should be + * resolved to /ipfs/// + */ + const pathParts = [] + + if (urlPath.length > 0) { + pathParts.push(urlPath) + } + + if (resolvedPath != null && resolvedPath.length > 0) { + pathParts.push(resolvedPath) + } + const path = pathParts.join('/') + + return { + protocol, + cid, + path, + query + } +} diff --git a/packages/verified-fetch/src/utils/tlru.ts b/packages/verified-fetch/src/utils/tlru.ts new file mode 100644 index 00000000..0556c0e6 --- /dev/null +++ b/packages/verified-fetch/src/utils/tlru.ts @@ -0,0 +1,52 @@ +import hashlru from 'hashlru' + +/** + * Time Aware Least Recent Used Cache + * + * @see https://arxiv.org/pdf/1801.00390 + */ +export class TLRU { + private readonly lru: ReturnType + + constructor (maxSize: number) { + this.lru = hashlru(maxSize) + } + + get (key: string): T | undefined { + const value = this.lru.get(key) + + if (value != null) { + if (value.expire != null && value.expire < Date.now()) { + this.lru.remove(key) + + return undefined + } + + return value.value + } + + return undefined + } + + set (key: string, value: T, ttl: number): void { + this.lru.set(key, { value, expire: Date.now() + ttl }) + } + + has (key: string): boolean { + const value = this.get(key) + + if (value != null) { + return true + } + + return false + } + + remove (key: string): void { + this.lru.remove(key) + } + + clear (): void { + this.lru.clear() + } +} diff --git a/packages/verified-fetch/src/utils/walk-path.ts b/packages/verified-fetch/src/utils/walk-path.ts new file mode 100644 index 00000000..45f2066e --- /dev/null +++ b/packages/verified-fetch/src/utils/walk-path.ts @@ -0,0 +1,34 @@ +import { walkPath as exporterWalk, type ExporterOptions, type ReadableStorage, type UnixFSEntry } from 'ipfs-unixfs-exporter' +import type { CID } from 'multiformats/cid' + +export interface PathWalkerOptions extends ExporterOptions { + +} +export interface PathWalkerResponse { + ipfsRoots: CID[] + terminalElement: UnixFSEntry + +} + +export interface PathWalkerFn { + (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise +} + +export async function walkPath (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise { + const ipfsRoots: CID[] = [] + let terminalElement: UnixFSEntry | undefined + + for await (const entry of exporterWalk(path, blockstore, options)) { + ipfsRoots.push(entry.cid) + terminalElement = entry + } + + if (terminalElement == null) { + throw new Error('No terminal element found') + } + + return { + ipfsRoots, + terminalElement + } +} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts new file mode 100644 index 00000000..7f659d4c --- /dev/null +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -0,0 +1,323 @@ +import { dagCbor as heliaDagCbor, type DAGCBOR } from '@helia/dag-cbor' +import { dagJson as heliaDagJson, type DAGJSON } from '@helia/dag-json' +import { ipns as heliaIpns, type IPNS } from '@helia/ipns' +import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' +import { json as heliaJson, type JSON } from '@helia/json' +import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' +import { code as dagCborCode } from '@ipld/dag-cbor' +import { code as dagJsonCode } from '@ipld/dag-json' +import { code as dagPbCode } from '@ipld/dag-pb' +import { code as jsonCode } from 'multiformats/codecs/json' +import { decode, code as rawCode } from 'multiformats/codecs/raw' +import { CustomProgressEvent } from 'progress-events' +import { getStreamAndContentType } from './utils/get-stream-and-content-type.js' +import { parseResource } from './utils/parse-resource.js' +import { walkPath, type PathWalkerFn } from './utils/walk-path.js' +import type { CIDDetail, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' +import type { Helia } from '@helia/interface' +import type { AbortOptions, Logger } from '@libp2p/interface' +import type { UnixFSEntry } from 'ipfs-unixfs-exporter' +import type { CID } from 'multiformats/cid' + +interface VerifiedFetchComponents { + helia: Helia + ipns?: IPNS + unixfs?: HeliaUnixFs + dagJson?: DAGJSON + json?: JSON + dagCbor?: DAGCBOR + pathWalker?: PathWalkerFn +} + +/** + * Potential future options for the VerifiedFetch constructor. + */ +// eslint-disable-next-line @typescript-eslint/no-empty-interface +interface VerifiedFetchInit { + +} + +interface FetchHandlerFunctionArg { + cid: CID + path: string + terminalElement?: UnixFSEntry + options?: Omit & AbortOptions +} + +interface FetchHandlerFunction { + (options: FetchHandlerFunctionArg): Promise +} + +function convertOptions (options?: VerifiedFetchOptions): (Omit & AbortOptions) | undefined { + if (options == null) { + return undefined + } + + let signal: AbortSignal | undefined + if (options?.signal === null) { + signal = undefined + } + return { + ...options, + signal + } +} + +export class VerifiedFetch { + private readonly helia: Helia + private readonly ipns: IPNS + private readonly unixfs: HeliaUnixFs + private readonly dagJson: DAGJSON + private readonly dagCbor: DAGCBOR + private readonly json: JSON + private readonly pathWalker: PathWalkerFn + private readonly log: Logger + + constructor ({ helia, ipns, unixfs, dagJson, json, dagCbor, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) { + this.helia = helia + this.log = helia.logger.forComponent('helia:verified-fetch') + this.ipns = ipns ?? heliaIpns(helia, { + resolvers: [ + dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'), + dnsJsonOverHttps('https://dns.google/resolve') + ] + }) + this.unixfs = unixfs ?? heliaUnixFs(helia) + this.dagJson = dagJson ?? heliaDagJson(helia) + this.json = json ?? heliaJson(helia) + this.dagCbor = dagCbor ?? heliaDagCbor(helia) + this.pathWalker = pathWalker ?? walkPath + this.log.trace('created VerifiedFetch instance') + } + + // handle vnd.ipfs.ipns-record + private async handleIPNSRecord ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + const response = new Response('vnd.ipfs.ipns-record support is not implemented', { status: 501 }) + response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header + return response + } + + // handle vnd.ipld.car + private async handleIPLDCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + const response = new Response('vnd.ipld.car support is not implemented', { status: 501 }) + response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header + return response + } + + private async handleDagJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + this.log.trace('fetching %c/%s', cid, path) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid: cid.toString(), path })) + const result = await this.dagJson.get(cid, { + signal: options?.signal, + onProgress: options?.onProgress + }) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: cid.toString(), path })) + const response = new Response(JSON.stringify(result), { status: 200 }) + response.headers.set('content-type', 'application/json') + return response + } + + private async handleJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + this.log.trace('fetching %c/%s', cid, path) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid: cid.toString(), path })) + const result: Record = await this.json.get(cid, { + signal: options?.signal, + onProgress: options?.onProgress + }) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: cid.toString(), path })) + const response = new Response(JSON.stringify(result), { status: 200 }) + response.headers.set('content-type', 'application/json') + return response + } + + private async handleDagCbor ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + this.log.trace('fetching %c/%s', cid, path) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid: cid.toString(), path })) + const result = await this.dagCbor.get(cid, { + signal: options?.signal, + onProgress: options?.onProgress + }) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: cid.toString(), path })) + const response = new Response(JSON.stringify(result), { status: 200 }) + response.headers.set('content-type', 'application/json') + return response + } + + private async handleDagPb ({ cid, path, options, terminalElement }: FetchHandlerFunctionArg): Promise { + this.log.trace('fetching %c/%s', cid, path) + let resolvedCID = terminalElement?.cid ?? cid + let stat: UnixFSStats + if (terminalElement?.type === 'directory') { + const dirCid = terminalElement.cid + + const rootFilePath = 'index.html' + try { + this.log.trace('found directory at %c/%s, looking for index.html', cid, path) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid: dirCid.toString(), path: rootFilePath })) + stat = await this.unixfs.stat(dirCid, { + path: rootFilePath, + signal: options?.signal, + onProgress: options?.onProgress + }) + this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid) + path = rootFilePath + resolvedCID = stat.cid + // terminalElement = stat + } catch (err: any) { + this.log('error loading path %c/%s', dirCid, rootFilePath, err) + return new Response('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented', { status: 501 }) + } finally { + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: dirCid.toString(), path: rootFilePath })) + } + } + + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid: resolvedCID.toString(), path: '' })) + const asyncIter = this.unixfs.cat(resolvedCID, { + signal: options?.signal, + onProgress: options?.onProgress + }) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: resolvedCID.toString(), path: '' })) + this.log('got async iterator for %c/%s', cid, path) + + const { contentType, stream } = await getStreamAndContentType(asyncIter, path ?? '', this.helia.logger, { + onProgress: options?.onProgress + }) + const response = new Response(stream, { status: 200 }) + response.headers.set('content-type', contentType) + + return response + } + + private async handleRaw ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + this.log.trace('fetching %c/%s', cid, path) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid: cid.toString(), path })) + const result = await this.helia.blockstore.get(cid) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: cid.toString(), path })) + const response = new Response(decode(result), { status: 200 }) + response.headers.set('content-type', 'application/octet-stream') + return response + } + + /** + * Determines the format requested by the client, defaults to `null` if no format is requested. + * + * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter + * @default 'raw' + */ + private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: string | null }): string | null { + const formatMap: Record = { + 'vnd.ipld.raw': 'raw', + 'vnd.ipld.car': 'car', + 'application/x-tar': 'tar', + 'application/vnd.ipld.dag-json': 'dag-json', + 'application/vnd.ipld.dag-cbor': 'dag-cbor', + 'application/json': 'json', + 'application/cbor': 'cbor', + 'vnd.ipfs.ipns-record': 'ipns-record' + } + + if (headerFormat != null) { + for (const format in formatMap) { + if (headerFormat.includes(format)) { + return formatMap[format] + } + } + } else if (queryFormat != null) { + return queryFormat + } + + return null + } + + /** + * Map of format to specific handlers for that format. + * These format handlers should adjust the response headers as specified in https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers + */ + private readonly formatHandlers: Record = { + raw: async () => new Response('application/vnd.ipld.raw support is not implemented', { status: 501 }), + car: this.handleIPLDCar, + 'ipns-record': this.handleIPNSRecord, + tar: async () => new Response('application/x-tar support is not implemented', { status: 501 }), + 'dag-json': async () => new Response('application/vnd.ipld.dag-json support is not implemented', { status: 501 }), + 'dag-cbor': async () => new Response('application/vnd.ipld.dag-cbor support is not implemented', { status: 501 }), + json: async () => new Response('application/json support is not implemented', { status: 501 }), + cbor: async () => new Response('application/cbor support is not implemented', { status: 501 }) + } + + private readonly codecHandlers: Record = { + [dagJsonCode]: this.handleDagJson, + [dagPbCode]: this.handleDagPb, + [jsonCode]: this.handleJson, + [dagCborCode]: this.handleDagCbor, + [rawCode]: this.handleRaw + } + + async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise { + const options = convertOptions(opts) + const { path, query, ...rest } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) + const cid = rest.cid + let response: Response | undefined + + const format = this.getFormat({ headerFormat: new Headers(options?.headers).get('accept'), queryFormat: query.format ?? null }) + + if (format != null) { + // TODO: These should be handled last when they're returning something other than 501 + const formatHandler = this.formatHandlers[format] + + if (formatHandler != null) { + response = await formatHandler.call(this, { cid, path, options }) + + if (response.status === 501) { + return response + } + } + } + + let terminalElement: UnixFSEntry | undefined + let ipfsRoots: CID[] | undefined + + try { + const pathDetails = await this.pathWalker(this.helia.blockstore, `${cid.toString()}/${path}`, options) + ipfsRoots = pathDetails.ipfsRoots + terminalElement = pathDetails.terminalElement + } catch (err) { + this.log.error('Error walking path %s', path, err) + // return new Response(`Error walking path: ${(err as Error).message}`, { status: 500 }) + } + + if (response == null) { + const codecHandler = this.codecHandlers[cid.code] + + if (codecHandler != null) { + response = await codecHandler.call(this, { cid, path, options, terminalElement }) + } else { + return new Response(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`, { status: 501 }) + } + } + + response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header + response.headers.set('cache-cotrol', 'public, max-age=29030400, immutable') + response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header + + if (ipfsRoots != null) { + response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header + } + // response.headers.set('Content-Disposition', `TODO`) // https://specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header + + return response + } + + /** + * Start the Helia instance + */ + async start (): Promise { + await this.helia.start() + } + + /** + * Shut down the Helia instance + */ + async stop (): Promise { + await this.helia.stop() + } +} diff --git a/packages/verified-fetch/test/get-content-type.spec.ts b/packages/verified-fetch/test/get-content-type.spec.ts new file mode 100644 index 00000000..a896945d --- /dev/null +++ b/packages/verified-fetch/test/get-content-type.spec.ts @@ -0,0 +1,34 @@ +import { expect } from 'aegir/chai' +import { getContentType } from '../src/utils/get-content-type.js' + +describe('get-content-type', () => { + it('should return image/svg+xml for svg input', async () => { + const input = { bytes: new TextEncoder().encode(''), path: 'test.svg' } + const output = await getContentType(input) + expect(output).to.equal('image/svg+xml') + }) + + it('should return image/svg+xml for svg input with xml header', async () => { + const input = { bytes: new TextEncoder().encode(''), path: 'test.svg' } + const output = await getContentType(input) + expect(output).to.equal('image/svg+xml') + }) + + it('should return audio/mpeg for "*.mp3"', async () => { + const input = { bytes: new Uint8Array(), path: 'foobar.mp3' } + const output = await getContentType(input) + expect(output).to.equal('audio/mpeg') + }) + + it('should return text/plain for "*.txt"', async () => { + const input = { bytes: new Uint8Array(), path: 'test.txt' } + const output = await getContentType(input) + expect(output).to.equal('text/plain') + }) + + it('should return default mime type', async () => { + const input = { bytes: new Uint8Array(), path: 'unrecognized' } + const output = await getContentType(input) + expect(output).to.equal('application/octet-stream') + }) +}) diff --git a/packages/verified-fetch/test/get-stream-and-content-type.spec.ts b/packages/verified-fetch/test/get-stream-and-content-type.spec.ts new file mode 100644 index 00000000..b270eb07 --- /dev/null +++ b/packages/verified-fetch/test/get-stream-and-content-type.spec.ts @@ -0,0 +1,68 @@ +import { defaultLogger } from '@libp2p/logger' +import { expect } from 'aegir/chai' +import sinon from 'sinon' +import { getStreamAndContentType } from '../src/utils/get-stream-and-content-type.js' + +describe('getStreamAndContentType', () => { + let onProgressSpy: sinon.SinonSpy + + beforeEach(() => { + onProgressSpy = sinon.spy() + }) + + it('should throw an error if no content is found', async () => { + const iterator = (async function * () { })() + await expect(getStreamAndContentType(iterator, 'test', defaultLogger())).to.be.rejectedWith('No content found') + }) + + it('should return the correct content type and a readable stream', async () => { + const iterator = (async function * () { yield new TextEncoder().encode('Hello, world!') })() + const { contentType, stream } = await getStreamAndContentType(iterator, 'test.txt', defaultLogger(), { onProgress: onProgressSpy }) + expect(contentType).to.equal('text/plain') + const reader = stream.getReader() + const { value } = await reader.read() + expect(onProgressSpy.callCount).to.equal(1) + expect(new TextDecoder().decode(value)).to.equal('Hello, world!') + }) + + it('should handle multiple chunks of data', async () => { + const iterator = (async function * () { yield new TextEncoder().encode('Hello,'); yield new TextEncoder().encode(' world!') })() + const { contentType, stream } = await getStreamAndContentType(iterator, 'test.txt', defaultLogger(), { onProgress: onProgressSpy }) + expect(contentType).to.equal('text/plain') + const reader = stream.getReader() + let result = '' + let chunk + while (!(chunk = await reader.read()).done) { + result += new TextDecoder().decode(chunk.value) + } + expect(onProgressSpy.callCount).to.equal(2) + expect(result).to.equal('Hello, world!') + }) + + it('should include last value done is true', async () => { + // if done === true and there is a value + const LIMIT = 5 + const iterator: AsyncIterable = { + [Symbol.asyncIterator] () { + let i = 0 + return { + async next () { + const done = i === LIMIT + const value = new Uint8Array([i++]) + return Promise.resolve({ value, done }) + } + } + } + } + const { contentType, stream } = await getStreamAndContentType(iterator, 'test.txt', defaultLogger(), { onProgress: onProgressSpy }) + expect(contentType).to.equal('text/plain') + const reader = stream.getReader() + const result = [] + let chunk + while (!(chunk = await reader.read()).done) { + result.push(...chunk.value) + } + expect(onProgressSpy.callCount).to.equal(6) + expect(result).to.deep.equal([...Array(LIMIT + 1).keys()]) + }) +}) diff --git a/packages/verified-fetch/test/index.spec.ts b/packages/verified-fetch/test/index.spec.ts new file mode 100644 index 00000000..1e0f47ef --- /dev/null +++ b/packages/verified-fetch/test/index.spec.ts @@ -0,0 +1,41 @@ +/* eslint-env mocha */ +import { createHeliaHTTP } from '@helia/http' +import { expect } from 'aegir/chai' +import { createHelia } from 'helia' +import { createVerifiedFetch } from '../src/index.js' + +describe('createVerifiedFetch', () => { + it('can be constructed with a HeliaHttp instance', async () => { + const heliaHttp = await createHeliaHTTP() + const verifiedFetch = await createVerifiedFetch(heliaHttp) + + expect(verifiedFetch).to.be.ok() + await verifiedFetch.stop() + }) + + it('can be constructed with a HeliaP2P instance', async () => { + const heliaP2P = await createHelia() + const verifiedFetch = await createVerifiedFetch(heliaP2P) + + expect(verifiedFetch).to.be.ok() + await heliaP2P.stop() + await verifiedFetch.stop() + }) + + it('can be constructed with gateways', async () => { + const verifiedFetch = await createVerifiedFetch({ + gateways: ['https://127.0.0.1'] + }) + expect(verifiedFetch).to.be.ok() + await verifiedFetch.stop() + }) + + it('can be constructed with gateways & routers', async () => { + const verifiedFetch = await createVerifiedFetch({ + gateways: ['https://127.0.0.1'], + routers: ['https://127.0.0.1'] + }) + expect(verifiedFetch).to.be.ok() + await verifiedFetch.stop() + }) +}) diff --git a/packages/verified-fetch/test/parse-resource.spec.ts b/packages/verified-fetch/test/parse-resource.spec.ts new file mode 100644 index 00000000..6e59817f --- /dev/null +++ b/packages/verified-fetch/test/parse-resource.spec.ts @@ -0,0 +1,32 @@ +import { defaultLogger } from '@libp2p/logger' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import sinon from 'sinon' +import { stubInterface } from 'sinon-ts' +import { parseResource } from '../src/utils/parse-resource.js' +import type { IPNS } from '@helia/ipns' + +describe('parseResource', () => { + it('does not call @helia/ipns for CID', async () => { + const testCID = CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + const shouldNotBeCalled1 = sinon.stub().throws(new Error('should not be called')) + const shouldNotBeCalled2 = sinon.stub().throws(new Error('should not be called')) + const { cid, path, query } = await parseResource(testCID, { + ipns: stubInterface({ + resolveDns: shouldNotBeCalled1, + resolve: shouldNotBeCalled2 + }), + logger: defaultLogger() + }) + expect(shouldNotBeCalled1.called).to.be.false() + expect(shouldNotBeCalled2.called).to.be.false() + expect(cid.toString()).to.equal(testCID.toString()) + expect(path).to.equal('') + expect(query).to.deep.equal({}) + }) + + it('throws an error if given an invalid resource', async () => { + // @ts-expect-error - purposefully invalid input + await expect(parseResource({}, stubInterface())).to.be.rejectedWith('Invalid resource.') + }) +}) diff --git a/packages/verified-fetch/test/parse-url-string.spec.ts b/packages/verified-fetch/test/parse-url-string.spec.ts new file mode 100644 index 00000000..6c56f112 --- /dev/null +++ b/packages/verified-fetch/test/parse-url-string.spec.ts @@ -0,0 +1,323 @@ +import { type PeerId } from '@libp2p/interface' +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { stubInterface } from 'sinon-ts' +import { parseUrlString } from '../src/utils/parse-url-string.js' +import type { IPNS } from '@helia/ipns' + +describe('parseUrlString', () => { + describe('invalid URLs', () => { + it('throws for invalid URLs', async () => { + const ipns = stubInterface({}) + try { + await parseUrlString({ + urlString: 'invalid', + ipns, + logger: defaultLogger() + }) + throw new Error('Should have thrown') + } catch (err) { + expect((err as Error).message).to.equal('Invalid URL: invalid, please use ipfs:// or ipns:// URLs only.') + } + }) + + it('throws for invalid protocols', async () => { + const ipns = stubInterface({}) + try { + await parseUrlString({ + urlString: 'http://mydomain.com', + ipns, + logger: defaultLogger() + }) + throw new Error('Should have thrown') + } catch (err) { + expect((err as Error).message).to.equal('Invalid URL: http://mydomain.com, please use ipfs:// or ipns:// URLs only.') + } + }) + + it('throws an error if resulting CID is invalid', async () => { + const ipns = stubInterface({ + // @ts-expect-error - purposefully invalid response + resolveDns: async (_: string) => { + return null + } + }) + try { + await parseUrlString({ + urlString: 'ipns://mydomain.com', + ipns, + logger: defaultLogger() + }) + throw new Error('Should have thrown') + } catch (err) { + expect((err as Error).message).to.equal('Invalid resource. Cannot determine CID from URL "ipns://mydomain.com"') + } + }) + }) + + describe('ipfs:// URLs', () => { + it('handles invalid CIDs', async () => { + const ipns = stubInterface({}) + try { + await parseUrlString({ + urlString: 'ipfs://QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4i', + ipns, + logger: defaultLogger() + }) + throw new Error('Should have thrown') + } catch (aggErr) { + expect(aggErr).to.have.property('message', 'Invalid resource. Cannot determine CID from URL "ipfs://QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4i"') + expect(aggErr).to.have.property('errors').with.lengthOf(1).that.deep.equals([ + new TypeError('Invalid CID for ipfs:// URL') + ]) + } + }) + + it('can parse a URL with CID only', async () => { + const ipns = stubInterface({}) + const result = await parseUrlString({ + urlString: 'ipfs://QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr', + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipfs') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('') + }) + + it('can parse URL with CID+path', async () => { + const ipns = stubInterface({}) + const result = await parseUrlString({ + urlString: 'ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt', + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipfs') + expect(result.cid.toString()).to.equal('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') + expect(result.path).to.equal('1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt') + }) + + it('can parse URL with CID+queryString', async () => { + const ipns = stubInterface({}) + const result = await parseUrlString({ + urlString: 'ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=car', + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipfs') + expect(result.cid.toString()).to.equal('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') + expect(result.path).to.equal('') + expect(result.query).to.deep.equal({ format: 'car' }) + }) + + it('can parse URL with CID+path+queryString', async () => { + const ipns = stubInterface({}) + const result = await parseUrlString({ + urlString: 'ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar', + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipfs') + expect(result.cid.toString()).to.equal('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') + expect(result.path).to.equal('1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt') + expect(result.query).to.deep.equal({ format: 'tar' }) + }) + }) + + describe('ipns:// URLs', () => { + let ipns: IPNS + + beforeEach(async () => { + ipns = stubInterface({ + resolveDns: async (dnsLink: string) => { + expect(dnsLink).to.equal('mydomain.com') + return { + cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), + path: '' + } + } + }) + }) + + it('handles invalid DNSLinkDomains', async () => { + ipns = stubInterface({ + resolve: async (peerId: PeerId) => { + throw new Error('Unexpected failure from ipns resolve method') + }, + resolveDns: async (_: string) => { + return Promise.reject(new Error('Unexpected failure from dns query')) + } + }) + + try { + await parseUrlString({ urlString: 'ipns://mydomain.com', ipns, logger: defaultLogger() }) + throw new Error('Should have thrown') + } catch (aggErr) { + expect(aggErr).to.have.property('message', 'Invalid resource. Cannot determine CID from URL "ipns://mydomain.com"') + expect(aggErr).to.have.property('errors').with.lengthOf(2).that.deep.equals([ + new TypeError('Could not parse PeerId in ipns url "mydomain.com", Non-base64 character'), + new Error('Unexpected failure from dns query') + ]) + } + }) + + it('can parse a URL with DNSLinkDomain only', async () => { + const result = await parseUrlString({ + urlString: 'ipns://mydomain.com', + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipns') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('') + }) + + it('can parse a URL with DNSLinkDomain+path', async () => { + const result = await parseUrlString({ + urlString: 'ipns://mydomain.com/some/path/to/file.txt', + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipns') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('some/path/to/file.txt') + }) + + it('can parse a URL with DNSLinkDomain+queryString', async () => { + const result = await parseUrlString({ + urlString: 'ipns://mydomain.com?format=json', + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipns') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('') + expect(result.query).to.deep.equal({ format: 'json' }) + }) + + it('can parse a URL with DNSLinkDomain+path+queryString', async () => { + const result = await parseUrlString({ + urlString: 'ipns://mydomain.com/some/path/to/file.txt?format=json', + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipns') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('some/path/to/file.txt') + expect(result.query).to.deep.equal({ format: 'json' }) + }) + }) + + describe('ipns:// URLs', () => { + let ipns: IPNS + let testPeerId: PeerId + + beforeEach(async () => { + testPeerId = await createEd25519PeerId() + ipns = stubInterface({ + resolve: async (peerId: PeerId) => { + expect(peerId.toString()).to.equal(testPeerId.toString()) + return { + cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), + path: '' + } + } + }) + }) + + it('handles invalid PeerIds', async () => { + ipns = stubInterface({ + resolve: async (peerId: PeerId) => { + throw new Error('Unexpected failure from ipns resolve method') + }, + resolveDns: async (_: string) => { + return Promise.reject(new Error('Unexpected failure from dns query')) + } + }) + + try { + await parseUrlString({ urlString: 'ipns://123PeerIdIsFake456', ipns, logger: defaultLogger() }) + throw new Error('Should have thrown') + } catch (aggErr) { + expect(aggErr).to.have.property('message', 'Invalid resource. Cannot determine CID from URL "ipns://123PeerIdIsFake456"') + expect(aggErr).to.have.property('errors').with.lengthOf(2).that.deep.equals([ + new TypeError('Could not parse PeerId in ipns url "123PeerIdIsFake456", Non-base58btc character'), + new Error('Unexpected failure from dns query') + ]) + } + }) + + it('handles valid PeerId resolve failures', async () => { + ipns = stubInterface({ + resolve: async (_: PeerId) => { + return Promise.reject(new Error('Unexpected failure from ipns resolve method')) + }, + resolveDns: async (_: string) => { + return Promise.reject(new Error('Unexpected failure from dns query')) + } + }) + + // await expect(parseUrlString({ urlString: `ipns://${testPeerId.toString()}`, ipns })).to.eventually.be.rejected() + // .with.property('message', `Could not resolve PeerId "${testPeerId.toString()}", Unexpected failure from ipns resolve method`) + + try { + await parseUrlString({ urlString: `ipns://${testPeerId.toString()}`, ipns, logger: defaultLogger() }) + throw new Error('Should have thrown') + } catch (aggErr) { + expect(aggErr).to.have.property('message', `Invalid resource. Cannot determine CID from URL "ipns://${testPeerId.toString()}"`) + expect(aggErr).to.have.property('errors').with.lengthOf(2).that.deep.equals([ + new TypeError(`Could not resolve PeerId "${testPeerId.toString()}", Unexpected failure from ipns resolve method`), + new Error('Unexpected failure from dns query') + ]) + } + }) + + it('can parse a URL with PeerId only', async () => { + const result = await parseUrlString({ + urlString: `ipns://${testPeerId.toString()}`, + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipns') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('') + }) + + it('can parse a URL with PeerId+path', async () => { + const result = await parseUrlString({ + urlString: `ipns://${testPeerId.toString()}/some/path/to/file.txt`, + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipns') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('some/path/to/file.txt') + }) + + it('can parse a URL with PeerId+queryString', async () => { + const result = await parseUrlString({ + urlString: `ipns://${testPeerId.toString()}?fomat=dag-cbor`, + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipns') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('') + expect(result.query).to.deep.equal({ fomat: 'dag-cbor' }) + }) + + it('can parse a URL with PeerId+path+queryString', async () => { + const result = await parseUrlString({ + urlString: `ipns://${testPeerId.toString()}/some/path/to/file.txt?fomat=dag-cbor`, + ipns, + logger: defaultLogger() + }) + expect(result.protocol).to.equal('ipns') + expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') + expect(result.path).to.equal('some/path/to/file.txt') + expect(result.query).to.deep.equal({ fomat: 'dag-cbor' }) + }) + }) +}) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts new file mode 100644 index 00000000..4dda9315 --- /dev/null +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -0,0 +1,398 @@ +/* eslint-env mocha */ +import { type DAGCBOR } from '@helia/dag-cbor' +import { type DAGJSON } from '@helia/dag-json' +import { type IPNS } from '@helia/ipns' +import { type JSON as HeliaJSON } from '@helia/json' +import { type UnixFS } from '@helia/unixfs' +import { defaultLogger } from '@libp2p/logger' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { encode } from 'multiformats/codecs/raw' +import sinon, { type SinonStub } from 'sinon' +import { stubInterface } from 'sinon-ts' +import { VerifiedFetch } from '../src/verified-fetch.js' +import type { PathWalkerFn } from '../src/utils/walk-path' +import type { Helia } from '@helia/interface' +import type { Logger, ComponentLogger } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' +import type { UnixFSDirectory, UnixFSEntry } from 'ipfs-unixfs-exporter' +const testCID = CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') +const anyOnProgressMatcher = sinon.match.any as unknown as () => void + +describe('@helia/verifed-fetch', () => { + it('starts and stops the helia node', async () => { + const stopStub = sinon.stub() + const startStub = sinon.stub() + const verifiedFetch = new VerifiedFetch({ + helia: stubInterface({ + start: startStub, + stop: stopStub, + logger: defaultLogger() + }) + }) + expect(stopStub.withArgs().callCount).to.equal(0) + expect(startStub.withArgs().callCount).to.equal(0) + await verifiedFetch.start() + expect(stopStub.withArgs().callCount).to.equal(0) + expect(startStub.withArgs().callCount).to.equal(1) + await verifiedFetch.stop() + expect(stopStub.withArgs().callCount).to.equal(1) + expect(startStub.withArgs().callCount).to.equal(1) + }) + + describe('format not implemented', () => { + let verifiedFetch: VerifiedFetch + + before(async () => { + verifiedFetch = new VerifiedFetch({ + helia: stubInterface({ + logger: stubInterface({ + forComponent: () => stubInterface() + }) + }), + ipns: stubInterface({ + resolveDns: async (dnsLink: string) => { + expect(dnsLink).to.equal('mydomain.com') + return { + cid: testCID, + path: '' + } + } + }), + unixfs: stubInterface() + }) + }) + + after(async () => { + await verifiedFetch.stop() + }) + + const formatsAndAcceptHeaders = [ + ['raw', 'application/vnd.ipld.raw'], + ['car', 'application/vnd.ipld.car'], + ['tar', 'application/x-tar'], + ['dag-json', 'application/vnd.ipld.dag-json'], + ['dag-cbor', 'application/vnd.ipld.dag-cbor'], + ['json', 'application/json'], + ['cbor', 'application/cbor'], + ['ipns-record', 'application/vnd.ipfs.ipns-record'] + ] + + for (const [format, acceptHeader] of formatsAndAcceptHeaders) { + // eslint-disable-next-line no-loop-func + it(`returns 501 for ${acceptHeader}`, async () => { + const resp = await verifiedFetch.fetch(`ipns://mydomain.com?format=${format}`) + expect(resp).to.be.ok() + expect(resp.status).to.equal(501) + const resp2 = await verifiedFetch.fetch(testCID, { + headers: { + accept: acceptHeader + } + }) + expect(resp2).to.be.ok() + expect(resp2.status).to.equal(501) + }) + } + }) + + describe('implicit format', () => { + let verifiedFetch: VerifiedFetch + let unixfsStub: ReturnType> + let dagJsonStub: ReturnType> + let jsonStub: ReturnType> + let dagCborStub: ReturnType> + let pathWalkerStub: SinonStub, ReturnType> + let blockstoreStub: ReturnType> + + beforeEach(async () => { + blockstoreStub = stubInterface() + unixfsStub = stubInterface({ + cat: sinon.stub(), + stat: sinon.stub() + }) + dagJsonStub = stubInterface({ + // @ts-expect-error - stub errors + get: sinon.stub() + }) + jsonStub = stubInterface({ + // @ts-expect-error - stub errors + get: sinon.stub() + }) + dagCborStub = stubInterface({ + // @ts-expect-error - stub errors + get: sinon.stub() + }) + pathWalkerStub = sinon.stub, ReturnType>() + verifiedFetch = new VerifiedFetch({ + helia: stubInterface({ + blockstore: blockstoreStub, + logger: defaultLogger() + }), + ipns: stubInterface(), + unixfs: unixfsStub, + dagJson: dagJsonStub, + json: jsonStub, + dagCbor: dagCborStub, + pathWalker: pathWalkerStub + }) + }) + + afterEach(async () => { + await verifiedFetch.stop() + }) + + it('should return raw data', async () => { + const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) + pathWalkerStub.returns(Promise.resolve({ + ipfsRoots: [testCID], + terminalElement: { + cid: testCID, + size: BigInt(3), + depth: 1, + content: async function * () { yield finalRootFileContent }, + name: 'index.html', + path: '', + type: 'raw', + node: finalRootFileContent + } + })) + unixfsStub.cat.returns({ + [Symbol.asyncIterator]: async function * () { + yield finalRootFileContent + } + }) + const resp = await verifiedFetch.fetch(testCID) + expect(pathWalkerStub.callCount).to.equal(1) + expect(unixfsStub.cat.callCount).to.equal(1) + expect(resp).to.be.ok() + expect(resp.status).to.equal(200) + const data = await resp.arrayBuffer() + expect(new Uint8Array(data)).to.deep.equal(finalRootFileContent) + }) + + it('should look for root files when directory is returned', async () => { + const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) + const signal = sinon.match.any as unknown as AbortSignal + const onProgress = sinon.spy() + // @ts-expect-error - stubbed type is incorrect + pathWalkerStub.onCall(0).returns(Promise.resolve({ + ipfsRoots: [testCID], + terminalElement: { + cid: testCID, + size: BigInt(3), + depth: 1, + // @ts-expect-error - stubbed type is incorrect + content: sinon.stub() as unknown as AsyncGenerator, + // @ts-expect-error - stubbed type is incorrect + unixfs: {} as unknown as UnixFS, + name: 'dirName', + path: '', + type: 'directory', + // @ts-expect-error - stubbed type is incorrect + node: {} + } satisfies UnixFSDirectory + })) + unixfsStub.stat.withArgs(testCID, { path: 'index.html', signal, onProgress: anyOnProgressMatcher }).onCall(0) + .returns(Promise.resolve({ + cid: CID.parse('Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi'), + size: 3, + type: 'raw', + fileSize: BigInt(3), + dagSize: BigInt(1), + localFileSize: BigInt(3), + localDagSize: BigInt(1), + blocks: 1 + })) + unixfsStub.cat.returns({ + [Symbol.asyncIterator]: async function * () { + yield finalRootFileContent + } + }) + unixfsStub.cat.returns({ + [Symbol.asyncIterator]: async function * () { + yield finalRootFileContent + } + }) + + const resp = await verifiedFetch.fetch(testCID, { onProgress }) + expect(unixfsStub.stat.callCount).to.equal(1) + expect(pathWalkerStub.callCount).to.equal(1) + expect(pathWalkerStub.getCall(0).args[1]).to.equal(`${testCID.toString()}/`) + expect(unixfsStub.cat.callCount).to.equal(1) + expect(unixfsStub.cat.withArgs(testCID).callCount).to.equal(0) + expect(unixfsStub.cat.withArgs(CID.parse('Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi'), sinon.match.any).callCount).to.equal(1) + expect(onProgress.callCount).to.equal(5) + + const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) + expect(onProgressEvents[0]).to.include({ type: 'verified-fetch:request:start' }).and.to.have.property('detail').that.deep.equals({ + cid: testCID.toString(), + path: 'index.html' + }) + expect(onProgressEvents[1]).to.include({ type: 'verified-fetch:request:end' }).and.to.have.property('detail').that.deep.equals({ + cid: testCID.toString(), + path: 'index.html' + }) + expect(onProgressEvents[3]).to.include({ type: 'verified-fetch:request:end' }).and.to.have.property('detail').that.deep.equals({ + cid: 'Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi', + path: '' + }) + expect(onProgressEvents[4]).to.include({ type: 'verified-fetch:request:progress:chunk' }).and.to.have.property('detail').that.is.undefined() + expect(resp).to.be.ok() + expect(resp.status).to.equal(200) + + const data = await resp.arrayBuffer() + expect(new Uint8Array(data)).to.deep.equal(finalRootFileContent) + }) + + it('should not call unixfs.cat if root file is not found', async () => { + const signal = sinon.match.any as unknown as AbortSignal + const onProgress = sinon.spy() + // @ts-expect-error - stubbed type is incorrect + pathWalkerStub.onCall(0).returns(Promise.resolve({ + ipfsRoots: [testCID], + terminalElement: { + cid: testCID, + size: BigInt(3), + depth: 1, + // @ts-expect-error - stubbed type is incorrect + content: sinon.stub() as unknown as AsyncGenerator, + // @ts-expect-error - stubbed type is incorrect + unixfs: {} as unknown as UnixFS, + name: 'dirName', + path: '', + type: 'directory', + // @ts-expect-error - stubbed type is incorrect + node: {} + } satisfies UnixFSDirectory + })) + + unixfsStub.stat.withArgs(testCID, { path: 'index.html', signal, onProgress: anyOnProgressMatcher }).onCall(0).throws(new Error('not found')) + const resp = await verifiedFetch.fetch(testCID) + + expect(unixfsStub.stat.withArgs(testCID, { path: 'index.html', signal, onProgress: anyOnProgressMatcher }).callCount).to.equal(1) + expect(unixfsStub.cat.withArgs(testCID).callCount).to.equal(0) + expect(onProgress.callCount).to.equal(0) + expect(resp).to.be.ok() + expect(resp.status).to.equal(501) + }) + + it('should return dag-json encoded CID', async () => { + const abortSignal = new AbortController().signal + const onProgress = sinon.spy() + const cid = CID.parse('baguqeerasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea') + dagJsonStub.get.withArgs(cid).returns(Promise.resolve({ + hello: 'world' + })) + const resp = await verifiedFetch.fetch(cid, { + signal: abortSignal, + onProgress + }) + expect(unixfsStub.stat.withArgs(cid).callCount).to.equal(0) + expect(unixfsStub.cat.withArgs(cid).callCount).to.equal(0) + expect(dagJsonStub.get.withArgs(cid).callCount).to.equal(1) + expect(onProgress.callCount).to.equal(2) + const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) + expect(onProgressEvents[0]).to.have.property('type', 'verified-fetch:request:start') + expect(onProgressEvents[0]).to.have.property('detail').that.deep.equals({ + cid: cid.toString(), + path: '' + }) + expect(onProgressEvents[1]).to.have.property('type', 'verified-fetch:request:end') + expect(onProgressEvents[1]).to.have.property('detail').that.deep.equals({ + cid: cid.toString(), + path: '' + }) + expect(resp).to.be.ok() + expect(resp.status).to.equal(200) + const data = await resp.json() + expect(data).to.deep.equal({ + hello: 'world' + }) + }) + + it('should return dag-cbor encoded CID', async () => { + const abortSignal = new AbortController().signal + const onProgress = sinon.spy() + const cid = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + dagCborStub.get.withArgs(cid).returns(Promise.resolve({ + hello: 'world' + })) + const resp = await verifiedFetch.fetch(cid, { + signal: abortSignal, + onProgress + }) + expect(resp).to.be.ok() + expect(resp.status).to.equal(200) + expect(unixfsStub.stat.withArgs(cid).callCount).to.equal(0) + expect(unixfsStub.cat.withArgs(cid).callCount).to.equal(0) + expect(dagCborStub.get.withArgs(cid).callCount).to.equal(1) + expect(onProgress.callCount).to.equal(2) + const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) + expect(onProgressEvents[0]).to.have.property('type', 'verified-fetch:request:start') + expect(onProgressEvents[0]).to.have.property('detail').that.deep.equals({ + cid: cid.toString(), + path: '' + }) + expect(onProgressEvents[1]).to.have.property('type', 'verified-fetch:request:end') + expect(onProgressEvents[1]).to.have.property('detail').that.deep.equals({ + cid: cid.toString(), + path: '' + }) + const data = await resp.json() + expect(data).to.deep.equal({ + hello: 'world' + }) + }) + + it('should return json encoded CID', async () => { + const abortSignal = new AbortController().signal + const onProgress = sinon.spy() + const cid = CID.parse('bagaaifcavabu6fzheerrmtxbbwv7jjhc3kaldmm7lbnvfopyrthcvod4m6ygpj3unrcggkzhvcwv5wnhc5ufkgzlsji7agnmofovc2g4a3ui7ja') + jsonStub.get.withArgs(cid).returns(Promise.resolve({ + hello: 'world' + })) + const resp = await verifiedFetch.fetch(cid, { + signal: abortSignal, + onProgress + }) + expect(unixfsStub.stat.withArgs(cid).callCount).to.equal(0) + expect(unixfsStub.cat.withArgs(cid).callCount).to.equal(0) + expect(dagJsonStub.get.withArgs(cid).callCount).to.equal(0) + expect(jsonStub.get.withArgs(cid).callCount).to.equal(1) + const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) + expect(onProgressEvents[0]).to.have.property('type', 'verified-fetch:request:start') + expect(onProgressEvents[0]).to.have.property('detail').that.deep.equals({ + cid: cid.toString(), + path: '' + }) + expect(onProgressEvents[1]).to.have.property('type', 'verified-fetch:request:end') + expect(onProgressEvents[1]).to.have.property('detail').that.deep.equals({ + cid: cid.toString(), + path: '' + }) + expect(resp).to.be.ok() + expect(resp.status).to.equal(200) + const data = await resp.json() + expect(data).to.deep.equal({ + hello: 'world' + }) + }) + + it('should handle raw identity CID', async () => { + const abortSignal = new AbortController().signal + const onProgress = sinon.spy() + const cid = CID.parse('bafkqac3imvwgy3zao5xxe3de') + const textEncoder = new TextEncoder() + blockstoreStub.get.withArgs(cid).returns(Promise.resolve(encode(textEncoder.encode('hello world')))) + const resp = await verifiedFetch.fetch(cid, { + signal: abortSignal, + onProgress + }) + expect(resp).to.be.ok() + // expect(resp.statusText).to.equal('OK') + expect(resp.status).to.equal(200) + const data = await resp.text() + expect(data).to.equal('hello world') + }) + }) +}) diff --git a/packages/verified-fetch/tsconfig.json b/packages/verified-fetch/tsconfig.json new file mode 100644 index 00000000..6c7cd39b --- /dev/null +++ b/packages/verified-fetch/tsconfig.json @@ -0,0 +1,42 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../block-brokers" + }, + { + "path": "../dag-cbor" + }, + { + "path": "../dag-json" + }, + { + "path": "../helia" + }, + { + "path": "../http" + }, + { + "path": "../interface" + }, + { + "path": "../ipns" + }, + { + "path": "../json" + }, + { + "path": "../routers" + }, + { + "path": "../unixfs" + } + ] +} diff --git a/packages/verified-fetch/typedoc.json b/packages/verified-fetch/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/verified-fetch/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +}