From 6eeb44dace9b17e8053a99c267650fd190b6ae74 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 12 Feb 2024 11:29:29 +0000 Subject: [PATCH 01/13] fix: remove stubs from verified-fetch tests The extensive stubbing in the `@helia/verified-fetch` tests have some baked-in assumptions about how the codecs work which are not easy to unpick. It's quick to test using the actual codecs if the block data is already present so remove the stubs and use a network-less Helia node to make the tests more reliable. --- packages/verified-fetch/package.json | 4 + packages/verified-fetch/src/verified-fetch.ts | 48 ++- .../test/fixtures/create-offline-helia.ts | 20 + .../test/verified-fetch.spec.ts | 396 ++++++------------ 4 files changed, 188 insertions(+), 280 deletions(-) create mode 100644 packages/verified-fetch/test/fixtures/create-offline-helia.ts diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 8945b835..03bd4686 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -161,12 +161,16 @@ "progress-events": "^1.0.0" }, "devDependencies": { + "@helia/utils": "^0.0.1", "@libp2p/logger": "^4.0.5", "@libp2p/peer-id-factory": "^4.0.5", "@sgtpooki/file-type": "^1.0.1", "@types/sinon": "^17.0.3", "aegir": "^42.2.2", + "blockstore-core": "^4.4.0", + "datastore-core": "^9.2.8", "helia": "^4.0.1", + "it-last": "^3.0.4", "magic-bytes.js": "^1.8.0", "sinon": "^17.0.1", "sinon-ts": "^2.0.0", diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index e513f46a..0719a6e8 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -9,6 +9,7 @@ import { code as dagJsonCode } from '@ipld/dag-json' import { code as dagPbCode } from '@ipld/dag-pb' import { code as jsonCode } from 'multiformats/codecs/json' import { decode, code as rawCode } from 'multiformats/codecs/raw' +import { identity } from 'multiformats/hashes/identity' import { CustomProgressEvent } from 'progress-events' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { parseResource } from './utils/parse-resource.js' @@ -62,6 +63,20 @@ function convertOptions (options?: VerifiedFetchOptions): (Omit { - const response = new Response('vnd.ipfs.ipns-record support is not implemented', { status: 501 }) + const response = notSupportedResponse('vnd.ipfs.ipns-record support is not implemented') response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header return response } // handle vnd.ipld.car private async handleIPLDCar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - const response = new Response('vnd.ipld.car support is not implemented', { status: 501 }) + const response = notSupportedResponse('vnd.ipld.car support is not implemented') response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header return response } @@ -113,7 +128,7 @@ export class VerifiedFetch { onProgress: options?.onProgress }) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - const response = new Response(JSON.stringify(result), { status: 200 }) + const response = okResponse(JSON.stringify(result)) response.headers.set('content-type', 'application/json') return response } @@ -126,7 +141,7 @@ export class VerifiedFetch { onProgress: options?.onProgress }) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - const response = new Response(JSON.stringify(result), { status: 200 }) + const response = okResponse(JSON.stringify(result)) response.headers.set('content-type', 'application/json') return response } @@ -139,7 +154,7 @@ export class VerifiedFetch { onProgress: options?.onProgress }) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - const response = new Response(result, { status: 200 }) + const response = okResponse(JSON.stringify(result)) await this.setContentType(result, path, response) return response } @@ -166,7 +181,7 @@ export class VerifiedFetch { // terminalElement = stat } catch (err: any) { this.log('error loading path %c/%s', dirCid, rootFilePath, err) - return new Response('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented', { status: 501 }) + return notSupportedResponse('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented') } finally { options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: dirCid, path: rootFilePath })) } @@ -183,7 +198,7 @@ export class VerifiedFetch { const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, { onProgress: options?.onProgress }) - const response = new Response(stream, { status: 200 }) + const response = okResponse(stream) await this.setContentType(firstChunk, path, response) return response @@ -194,7 +209,7 @@ export class VerifiedFetch { options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) const result = await this.helia.blockstore.get(cid) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - const response = new Response(decode(result), { status: 200 }) + const response = okResponse(decode(result)) await this.setContentType(result, path, response) return response } @@ -261,14 +276,14 @@ export class VerifiedFetch { * These format handlers should adjust the response headers as specified in https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers */ private readonly formatHandlers: Record = { - raw: async () => new Response('application/vnd.ipld.raw support is not implemented', { status: 501 }), + raw: async () => notSupportedResponse('application/vnd.ipld.raw support is not implemented'), car: this.handleIPLDCar, 'ipns-record': this.handleIPNSRecord, - tar: async () => new Response('application/x-tar support is not implemented', { status: 501 }), - 'dag-json': async () => new Response('application/vnd.ipld.dag-json support is not implemented', { status: 501 }), - 'dag-cbor': async () => new Response('application/vnd.ipld.dag-cbor support is not implemented', { status: 501 }), - json: async () => new Response('application/json support is not implemented', { status: 501 }), - cbor: async () => new Response('application/cbor support is not implemented', { status: 501 }) + tar: async () => notSupportedResponse('application/x-tar support is not implemented'), + 'dag-json': async () => notSupportedResponse('application/vnd.ipld.dag-json support is not implemented'), + 'dag-cbor': async () => notSupportedResponse('application/vnd.ipld.dag-cbor support is not implemented'), + json: async () => notSupportedResponse('application/json support is not implemented'), + cbor: async () => notSupportedResponse('application/cbor support is not implemented') } private readonly codecHandlers: Record = { @@ -276,7 +291,8 @@ export class VerifiedFetch { [dagPbCode]: this.handleDagPb, [jsonCode]: this.handleJson, [dagCborCode]: this.handleDagCbor, - [rawCode]: this.handleRaw + [rawCode]: this.handleRaw, + [identity.code]: this.handleRaw } async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise { @@ -318,7 +334,7 @@ export class VerifiedFetch { if (codecHandler != null) { response = await codecHandler.call(this, { cid, path, options, terminalElement }) } else { - return new Response(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`, { status: 501 }) + return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`) } } diff --git a/packages/verified-fetch/test/fixtures/create-offline-helia.ts b/packages/verified-fetch/test/fixtures/create-offline-helia.ts new file mode 100644 index 00000000..2c6ab4af --- /dev/null +++ b/packages/verified-fetch/test/fixtures/create-offline-helia.ts @@ -0,0 +1,20 @@ +import { Helia as HeliaClass } from '@helia/utils' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' + +export async function createHelia (): Promise { + const datastore = new MemoryDatastore() + const blockstore = new MemoryBlockstore() + + const helia = new HeliaClass({ + datastore, + blockstore, + blockBrokers: [], + routers: [] + }) + + await helia.start() + + return helia +} diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index b36cfc48..77a4b256 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -1,43 +1,56 @@ /* eslint-env mocha */ -import { type DAGCBOR } from '@helia/dag-cbor' -import { type DAGJSON } from '@helia/dag-json' +import { dagCbor } from '@helia/dag-cbor' +import { dagJson } from '@helia/dag-json' import { type IPNS } from '@helia/ipns' -import { type JSON as HeliaJSON } from '@helia/json' -import { type UnixFS } from '@helia/unixfs' +import { json } from '@helia/json' +import { unixfs, type UnixFS } from '@helia/unixfs' +import { stop } from '@libp2p/interface' import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' +import last from 'it-last' import { CID } from 'multiformats/cid' -import { encode } from 'multiformats/codecs/raw' -import sinon, { type SinonStub } from 'sinon' +import * as raw from 'multiformats/codecs/raw' +import { identity } from 'multiformats/hashes/identity' +import { sha256 } from 'multiformats/hashes/sha2' +import Sinon from 'sinon' import { stubInterface } from 'sinon-ts' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { VerifiedFetch } from '../src/verified-fetch.js' -import type { PathWalkerFn } from '../src/utils/walk-path' -import type { Blocks, Helia } from '@helia/interface' +import { createHelia } from './fixtures/create-offline-helia.js' +import type { Helia } from '@helia/interface' import type { Logger, ComponentLogger } from '@libp2p/interface' -import type { UnixFSDirectory, UnixFSEntry } from 'ipfs-unixfs-exporter' const testCID = CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') -const anyOnProgressMatcher = sinon.match.any as unknown as () => void describe('@helia/verifed-fetch', () => { + let helia: Helia + + beforeEach(async () => { + helia = await createHelia() + }) + + afterEach(async () => { + await stop(helia) + }) + it('starts and stops the helia node', async () => { - const stopStub = sinon.stub() - const startStub = sinon.stub() + const helia = stubInterface({ + logger: defaultLogger() + }) const verifiedFetch = new VerifiedFetch({ - helia: stubInterface({ - start: startStub, - stop: stopStub, - logger: defaultLogger() - }) + helia }) - expect(stopStub.withArgs().callCount).to.equal(0) - expect(startStub.withArgs().callCount).to.equal(0) + + expect(helia.stop.callCount).to.equal(0) + expect(helia.start.callCount).to.equal(0) + await verifiedFetch.start() - expect(stopStub.withArgs().callCount).to.equal(0) - expect(startStub.withArgs().callCount).to.equal(1) + expect(helia.stop.callCount).to.equal(0) + expect(helia.start.callCount).to.equal(1) + await verifiedFetch.stop() - expect(stopStub.withArgs().callCount).to.equal(1) - expect(startStub.withArgs().callCount).to.equal(1) + expect(helia.stop.callCount).to.equal(1) + expect(helia.start.callCount).to.equal(1) }) describe('format not implemented', () => { @@ -97,43 +110,10 @@ describe('@helia/verifed-fetch', () => { describe('implicit format', () => { let verifiedFetch: VerifiedFetch - let unixfsStub: ReturnType> - let dagJsonStub: ReturnType> - let jsonStub: ReturnType> - let dagCborStub: ReturnType> - let pathWalkerStub: SinonStub, ReturnType> - let blockstoreStub: ReturnType> beforeEach(async () => { - blockstoreStub = stubInterface() - unixfsStub = stubInterface({ - cat: sinon.stub(), - stat: sinon.stub() - }) - dagJsonStub = stubInterface({ - // @ts-expect-error - stub errors - get: sinon.stub() - }) - jsonStub = stubInterface({ - // @ts-expect-error - stub errors - get: sinon.stub() - }) - dagCborStub = stubInterface({ - // @ts-expect-error - stub errors - get: sinon.stub() - }) - pathWalkerStub = sinon.stub, ReturnType>() verifiedFetch = new VerifiedFetch({ - helia: stubInterface({ - blockstore: blockstoreStub, - logger: defaultLogger() - }), - ipns: stubInterface(), - unixfs: unixfsStub, - dagJson: dagJsonStub, - json: jsonStub, - dagCbor: dagCborStub, - pathWalker: pathWalkerStub + helia }) }) @@ -143,256 +123,144 @@ describe('@helia/verifed-fetch', () => { it('should return raw data', async () => { const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) - pathWalkerStub.returns(Promise.resolve({ - ipfsRoots: [testCID], - terminalElement: { - cid: testCID, - size: BigInt(3), - depth: 1, - content: async function * () { yield finalRootFileContent }, - name: 'index.html', - path: '', - type: 'raw', - node: finalRootFileContent - } - })) - unixfsStub.cat.returns({ - [Symbol.asyncIterator]: async function * () { - yield finalRootFileContent - } - }) - const resp = await verifiedFetch.fetch(testCID) - expect(pathWalkerStub.callCount).to.equal(1) - expect(unixfsStub.cat.callCount).to.equal(1) + const cid = CID.createV1(raw.code, await sha256.digest(finalRootFileContent)) + await helia.blockstore.put(cid, finalRootFileContent) + + const resp = await verifiedFetch.fetch(`ipfs://${cid}`) expect(resp).to.be.ok() expect(resp.status).to.equal(200) + expect(resp.statusText).to.equal('OK') const data = await resp.arrayBuffer() - expect(new Uint8Array(data)).to.deep.equal(finalRootFileContent) + expect(new Uint8Array(data)).to.equalBytes(finalRootFileContent) }) - it('should look for root files when directory is returned', async () => { + it('should report progress during fetch', async () => { const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) - const signal = sinon.match.any as unknown as AbortSignal - const onProgress = sinon.spy() - // @ts-expect-error - stubbed type is incorrect - pathWalkerStub.onCall(0).returns(Promise.resolve({ - ipfsRoots: [testCID], - terminalElement: { - cid: testCID, - size: BigInt(3), - depth: 1, - // @ts-expect-error - stubbed type is incorrect - content: sinon.stub() as unknown as AsyncGenerator, - // @ts-expect-error - stubbed type is incorrect - unixfs: {} as unknown as UnixFS, - name: 'dirName', - path: '', - type: 'directory', - // @ts-expect-error - stubbed type is incorrect - node: {} - } satisfies UnixFSDirectory - })) - unixfsStub.stat.withArgs(testCID, { path: 'index.html', signal, onProgress: anyOnProgressMatcher }).onCall(0) - .returns(Promise.resolve({ - cid: CID.parse('Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi'), - size: 3, - type: 'raw', - fileSize: BigInt(3), - dagSize: BigInt(1), - localFileSize: BigInt(3), - localDagSize: BigInt(1), - blocks: 1 - })) - unixfsStub.cat.returns({ - [Symbol.asyncIterator]: async function * () { - yield finalRootFileContent - } - }) - unixfsStub.cat.returns({ - [Symbol.asyncIterator]: async function * () { - yield finalRootFileContent - } + const cid = CID.createV1(raw.code, await sha256.digest(finalRootFileContent)) + await helia.blockstore.put(cid, finalRootFileContent) + + const onProgress = Sinon.spy() + + await verifiedFetch.fetch(`ipfs://${cid}`, { + onProgress }) - const resp = await verifiedFetch.fetch(testCID, { onProgress }) - expect(unixfsStub.stat.callCount).to.equal(1) - expect(pathWalkerStub.callCount).to.equal(1) - expect(pathWalkerStub.getCall(0).args[1]).to.equal(`${testCID.toString()}/`) - expect(unixfsStub.cat.callCount).to.equal(1) - expect(unixfsStub.cat.withArgs(testCID).callCount).to.equal(0) - expect(unixfsStub.cat.withArgs(CID.parse('Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi'), sinon.match.any).callCount).to.equal(1) - expect(onProgress.callCount).to.equal(5) + expect(onProgress.callCount).to.equal(3) const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) - expect(onProgressEvents[0]).to.include({ type: 'verified-fetch:request:start' }).and.to.have.property('detail').that.deep.equals({ - cid: testCID, - path: 'index.html' - }) - expect(onProgressEvents[1]).to.include({ type: 'verified-fetch:request:end' }).and.to.have.property('detail').that.deep.equals({ - cid: testCID, - path: 'index.html' + expect(onProgressEvents[0]).to.include({ type: 'blocks:get:blockstore:get' }).and.to.have.property('detail').that.deep.equals(cid) + expect(onProgressEvents[1]).to.include({ type: 'verified-fetch:request:start' }).and.to.have.property('detail').that.deep.equals({ + cid, + path: '' }) - expect(onProgressEvents[3]).to.include({ type: 'verified-fetch:request:end' }).and.to.have.property('detail').that.deep.equals({ - cid: CID.parse('Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi'), + expect(onProgressEvents[2]).to.include({ type: 'verified-fetch:request:end' }).and.to.have.property('detail').that.deep.equals({ + cid, path: '' }) - expect(onProgressEvents[4]).to.include({ type: 'verified-fetch:request:progress:chunk' }).and.to.have.property('detail').that.is.undefined() + }) + + it('should look for index files when directory is returned', async () => { + const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) + + const fs = unixfs(helia) + const res = await last(fs.addAll([{ + path: 'index.html', + content: finalRootFileContent + }], { + wrapWithDirectory: true + })) + + if (res == null) { + throw new Error('Import failed') + } + + const stat = await fs.stat(res.cid) + expect(stat.type).to.equal('directory') + + const resp = await verifiedFetch.fetch(res.cid) expect(resp).to.be.ok() expect(resp.status).to.equal(200) + expect(resp.statusText).to.equal('OK') const data = await resp.arrayBuffer() - expect(new Uint8Array(data)).to.deep.equal(finalRootFileContent) + expect(new Uint8Array(data)).to.equalBytes(finalRootFileContent) }) - it('should not call unixfs.cat if root file is not found', async () => { - const signal = sinon.match.any as unknown as AbortSignal - const onProgress = sinon.spy() - // @ts-expect-error - stubbed type is incorrect - pathWalkerStub.onCall(0).returns(Promise.resolve({ - ipfsRoots: [testCID], - terminalElement: { - cid: testCID, - size: BigInt(3), - depth: 1, - // @ts-expect-error - stubbed type is incorrect - content: sinon.stub() as unknown as AsyncGenerator, - // @ts-expect-error - stubbed type is incorrect - unixfs: {} as unknown as UnixFS, - name: 'dirName', - path: '', - type: 'directory', - // @ts-expect-error - stubbed type is incorrect - node: {} - } satisfies UnixFSDirectory + it('should return 501 if index file is not found', async () => { + const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) + + const fs = unixfs(helia) + const res = await last(fs.addAll([{ + path: 'not_an_index.html', + content: finalRootFileContent + }], { + wrapWithDirectory: true })) - unixfsStub.stat.withArgs(testCID, { path: 'index.html', signal, onProgress: anyOnProgressMatcher }).onCall(0).throws(new Error('not found')) - const resp = await verifiedFetch.fetch(testCID) + if (res == null) { + throw new Error('Import failed') + } - expect(unixfsStub.stat.withArgs(testCID, { path: 'index.html', signal, onProgress: anyOnProgressMatcher }).callCount).to.equal(1) - expect(unixfsStub.cat.withArgs(testCID).callCount).to.equal(0) - expect(onProgress.callCount).to.equal(0) + const stat = await fs.stat(res.cid) + expect(stat.type).to.equal('directory') + + const resp = await verifiedFetch.fetch(res.cid) expect(resp).to.be.ok() expect(resp.status).to.equal(501) + expect(resp.statusText).to.equal('Not Implemented') }) - it('should return dag-json encoded CID', async () => { - const abortSignal = new AbortController().signal - const onProgress = sinon.spy() - const cid = CID.parse('baguqeerasords4njcts6vs7qvdjfcvgnume4hqohf65zsfguprqphs3icwea') - dagJsonStub.get.withArgs(cid).returns(Promise.resolve({ + it('should handle dag-json block', async () => { + const obj = { hello: 'world' - })) - const resp = await verifiedFetch.fetch(cid, { - signal: abortSignal, - onProgress - }) - expect(unixfsStub.stat.withArgs(cid).callCount).to.equal(0) - expect(unixfsStub.cat.withArgs(cid).callCount).to.equal(0) - expect(dagJsonStub.get.withArgs(cid).callCount).to.equal(1) - expect(onProgress.callCount).to.equal(2) - const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) - expect(onProgressEvents[0]).to.have.property('type', 'verified-fetch:request:start') - expect(onProgressEvents[0]).to.have.property('detail').that.deep.equals({ - cid, - path: '' - }) - expect(onProgressEvents[1]).to.have.property('type', 'verified-fetch:request:end') - expect(onProgressEvents[1]).to.have.property('detail').that.deep.equals({ - cid, - path: '' - }) + } + const j = dagJson(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid) expect(resp).to.be.ok() expect(resp.status).to.equal(200) - const data = await resp.json() - expect(data).to.deep.equal({ - hello: 'world' - }) + expect(resp.statusText).to.equal('OK') + await expect(resp.json()).to.eventually.deep.equal(obj) }) - it('should return dag-cbor encoded CID', async () => { - const abortSignal = new AbortController().signal - const onProgress = sinon.spy() - const cid = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') - dagCborStub.get.withArgs(cid).returns(Promise.resolve(JSON.stringify({ + it('should handle dag-cbor block', async () => { + const obj = { hello: 'world' - }))) - const resp = await verifiedFetch.fetch(cid, { - signal: abortSignal, - onProgress - }) + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid) expect(resp).to.be.ok() expect(resp.status).to.equal(200) - expect(unixfsStub.stat.withArgs(cid).callCount).to.equal(0) - expect(unixfsStub.cat.withArgs(cid).callCount).to.equal(0) - expect(dagCborStub.get.withArgs(cid).callCount).to.equal(1) - expect(onProgress.callCount).to.equal(2) - const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) - expect(onProgressEvents[0]).to.have.property('type', 'verified-fetch:request:start') - expect(onProgressEvents[0]).to.have.property('detail').that.deep.equals({ - cid, - path: '' - }) - expect(onProgressEvents[1]).to.have.property('type', 'verified-fetch:request:end') - expect(onProgressEvents[1]).to.have.property('detail').that.deep.equals({ - cid, - path: '' - }) - const data = await resp.json() - expect(data).to.deep.equal({ - hello: 'world' - }) + expect(resp.statusText).to.equal('OK') + await expect(resp.json()).to.eventually.deep.equal(obj) }) - it('should return json encoded CID', async () => { - const abortSignal = new AbortController().signal - const onProgress = sinon.spy() - const cid = CID.parse('bagaaifcavabu6fzheerrmtxbbwv7jjhc3kaldmm7lbnvfopyrthcvod4m6ygpj3unrcggkzhvcwv5wnhc5ufkgzlsji7agnmofovc2g4a3ui7ja') - jsonStub.get.withArgs(cid).returns(Promise.resolve({ + it('should handle json block', async () => { + const obj = { hello: 'world' - })) - const resp = await verifiedFetch.fetch(cid, { - signal: abortSignal, - onProgress - }) - expect(unixfsStub.stat.withArgs(cid).callCount).to.equal(0) - expect(unixfsStub.cat.withArgs(cid).callCount).to.equal(0) - expect(dagJsonStub.get.withArgs(cid).callCount).to.equal(0) - expect(jsonStub.get.withArgs(cid).callCount).to.equal(1) - const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) - expect(onProgressEvents[0]).to.have.property('type', 'verified-fetch:request:start') - expect(onProgressEvents[0]).to.have.property('detail').that.deep.equals({ - cid, - path: '' - }) - expect(onProgressEvents[1]).to.have.property('type', 'verified-fetch:request:end') - expect(onProgressEvents[1]).to.have.property('detail').that.deep.equals({ - cid, - path: '' - }) + } + const j = json(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid) expect(resp).to.be.ok() expect(resp.status).to.equal(200) - const data = await resp.json() - expect(data).to.deep.equal({ - hello: 'world' - }) + expect(resp.statusText).to.equal('OK') + await expect(resp.json()).to.eventually.deep.equal(obj) }) - it('should handle raw identity CID', async () => { - const abortSignal = new AbortController().signal - const onProgress = sinon.spy() - const cid = CID.parse('bafkqac3imvwgy3zao5xxe3de') - const textEncoder = new TextEncoder() - blockstoreStub.get.withArgs(cid).returns(Promise.resolve(encode(textEncoder.encode('hello world')))) - const resp = await verifiedFetch.fetch(cid, { - signal: abortSignal, - onProgress - }) + it('should handle identity CID', async () => { + const data = uint8ArrayFromString('hello world') + const cid = CID.createV1(identity.code, identity.digest(data)) + + const resp = await verifiedFetch.fetch(cid) expect(resp).to.be.ok() - // expect(resp.statusText).to.equal('OK') expect(resp.status).to.equal(200) - const data = await resp.text() - expect(data).to.equal('hello world') + expect(resp.statusText).to.equal('OK') + await expect(resp.text()).to.eventually.equal('hello world') }) }) }) From 95e56188db891f5c30ce4999558fdb82ee72f08b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Feb 2024 16:59:33 +0100 Subject: [PATCH 02/13] fix: round-trip DAG-JSON and DAG-CBOR with embedded CIDs DAG-JSON and DAG-CBOR can have embeded CIDs which are deserialized to CID objects by the `@ipld/dag-json` and `@ipld/dag-cbor` modules. This fixes a bug whereby we were JSON.stringifying them which lead to rubbish being returned from `.json()`. --- packages/verified-fetch/src/verified-fetch.ts | 12 +++++-- .../test/verified-fetch.spec.ts | 32 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 0719a6e8..f36f39d6 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -128,7 +128,11 @@ export class VerifiedFetch { onProgress: options?.onProgress }) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - const response = okResponse(JSON.stringify(result)) + // return body as binary + const body = await this.helia.blockstore.get(cid) + const response = okResponse(body) + // return pre-parsed object with embedded CIDs as objects + response.json = async () => result response.headers.set('content-type', 'application/json') return response } @@ -154,7 +158,11 @@ export class VerifiedFetch { onProgress: options?.onProgress }) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - const response = okResponse(JSON.stringify(result)) + // return body as binary + const body = await this.helia.blockstore.get(cid) + const response = okResponse(body) + // return pre-parsed object with embedded CIDs as objects + response.json = async () => result await this.setContentType(result, path, response) return response } diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 77a4b256..938f9c35 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -224,6 +224,22 @@ describe('@helia/verifed-fetch', () => { await expect(resp.json()).to.eventually.deep.equal(obj) }) + it('should return dag-json data with embedded CID', async () => { + const obj = { + hello: 'world', + link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + } + const j = dagJson(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid) + const data = await resp.json() + expect(data).to.deep.equal({ + hello: 'world', + link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + }) + }) + it('should handle dag-cbor block', async () => { const obj = { hello: 'world' @@ -238,6 +254,22 @@ describe('@helia/verifed-fetch', () => { await expect(resp.json()).to.eventually.deep.equal(obj) }) + it('should return dag-cbor data with embedded CID', async () => { + const obj = { + hello: 'world', + link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid) + const data = await resp.json() + expect(data).to.deep.equal({ + hello: 'world', + link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + }) + }) + it('should handle json block', async () => { const obj = { hello: 'world' From 74f15ba03243d48742646d5960b0d548a1bd816d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 14 Feb 2024 10:22:31 +0000 Subject: [PATCH 03/13] chore: fix up cbor handling --- .../src/utils/dag-cbor-to-safe-json.ts | 58 +++++++++++ packages/verified-fetch/src/verified-fetch.ts | 78 ++++++--------- .../test/verified-fetch.spec.ts | 95 ++++++++++++++++++- 3 files changed, 179 insertions(+), 52 deletions(-) create mode 100644 packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts diff --git a/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts b/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts new file mode 100644 index 00000000..a6588809 --- /dev/null +++ b/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts @@ -0,0 +1,58 @@ +import * as dagJson from '@ipld/dag-json' +import { decode } from 'cborg' +import { CID } from 'multiformats/cid' +import type { TagDecoder } from 'cborg' + +// https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692 +const CID_CBOR_TAG = 0x2A + +function cidDecoder (bytes: Uint8Array): CID { + if (bytes[0] !== 0) { + throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00') + } + + return CID.decode(bytes.subarray(1)) // ignore leading 0x00 +} + +/** + * Take a `DAG-CBOR` encoded `Uint8Array`, deserialize it as an object and + * re-serialize it in a form that can be passed to `JSON.serialize` and then + * `JSON.parse` without losing any data. + * + * This is not as simple as it sounds because `DAG-CBOR` will return `BigInt`s + * for numbers with a value greater than `Number.MAX_SAFE_INTEGER` - `BigInt`s + * are not part of JSON so we disable support for them which will cause cborg + * to throw. + * + * `DAG-JSON` uses `Number`s exclusively even though JavaScript cannot safely + * decode them once they become greater than `Number.MAX_SAFE_INTEGER` so we + * doubly need to throw when they are encountered which will set the response + * type as `application/octet-stream` and the user can use `@ipld/dag-cbor` to + * decode the values in a safe way. + * + * `CID`s are re-encoded as `{ "/": "QmFoo" }` and `Uint8Array`s to + * `{ "/": { "bytes": "base64EncodedBytes" }}` as per the `DAG-JSON` spec. + */ +export function dagCborToSafeJSON (buf: Uint8Array): string { + const tags: TagDecoder[] = [] + tags[CID_CBOR_TAG] = cidDecoder + + const obj = decode(buf, { + allowIndefinite: false, + coerceUndefinedToNull: true, + allowNaN: false, + allowInfinity: false, + strict: true, + useMaps: false, + rejectDuplicateMapKeys: true, + tags, + + // this is different to `DAG-CBOR` - the reason we disallow BigInts is + // because we are about to re-encode to `DAG-JSON` which allows larger + // numbers than the JS Number type supports so we cause cborg to throw in + // order to prompt the user to decode using a method that preserves data. + allowBigInt: false + }) + + return new TextDecoder().decode(dagJson.encode(obj)) +} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index f36f39d6..c4c136ba 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -1,16 +1,15 @@ -import { dagCbor as heliaDagCbor, type DAGCBOR } from '@helia/dag-cbor' -import { dagJson as heliaDagJson, type DAGJSON } from '@helia/dag-json' import { ipns as heliaIpns, type IPNS } from '@helia/ipns' import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' import { json as heliaJson, type JSON } from '@helia/json' import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' -import { code as dagCborCode } from '@ipld/dag-cbor' -import { code as dagJsonCode } from '@ipld/dag-json' -import { code as dagPbCode } from '@ipld/dag-pb' -import { code as jsonCode } from 'multiformats/codecs/json' -import { decode, code as rawCode } from 'multiformats/codecs/raw' +import * as dagCbor from '@ipld/dag-cbor' +import * as dagJson from '@ipld/dag-json' +import * as dagPb from '@ipld/dag-pb' +import * as json from 'multiformats/codecs/json' +import * as raw from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import { CustomProgressEvent } from 'progress-events' +import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { parseResource } from './utils/parse-resource.js' import { walkPath, type PathWalkerFn } from './utils/walk-path.js' @@ -24,9 +23,7 @@ interface VerifiedFetchComponents { helia: Helia ipns?: IPNS unixfs?: HeliaUnixFs - dagJson?: DAGJSON json?: JSON - dagCbor?: DAGCBOR pathWalker?: PathWalkerFn } @@ -81,14 +78,12 @@ export class VerifiedFetch { private readonly helia: Helia private readonly ipns: IPNS private readonly unixfs: HeliaUnixFs - private readonly dagJson: DAGJSON - private readonly dagCbor: DAGCBOR private readonly json: JSON private readonly pathWalker: PathWalkerFn private readonly log: Logger private readonly contentTypeParser: ContentTypeParser | undefined - constructor ({ helia, ipns, unixfs, dagJson, json, dagCbor, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) { + constructor ({ helia, ipns, unixfs, json, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) { this.helia = helia this.log = helia.logger.forComponent('helia:verified-fetch') this.ipns = ipns ?? heliaIpns(helia, { @@ -98,9 +93,7 @@ export class VerifiedFetch { ] }) this.unixfs = unixfs ?? heliaUnixFs(helia) - this.dagJson = dagJson ?? heliaDagJson(helia) this.json = json ?? heliaJson(helia) - this.dagCbor = dagCbor ?? heliaDagCbor(helia) this.pathWalker = pathWalker ?? walkPath this.contentTypeParser = init?.contentTypeParser this.log.trace('created VerifiedFetch instance') @@ -120,23 +113,6 @@ export class VerifiedFetch { return response } - private async handleDagJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - this.log.trace('fetching %c/%s', cid, path) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) - const result = await this.dagJson.get(cid, { - signal: options?.signal, - onProgress: options?.onProgress - }) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - // return body as binary - const body = await this.helia.blockstore.get(cid) - const response = okResponse(body) - // return pre-parsed object with embedded CIDs as objects - response.json = async () => result - response.headers.set('content-type', 'application/json') - return response - } - private async handleJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) @@ -144,26 +120,28 @@ export class VerifiedFetch { signal: options?.signal, onProgress: options?.onProgress }) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) const response = okResponse(JSON.stringify(result)) response.headers.set('content-type', 'application/json') + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) return response } private async handleDagCbor ({ cid, path, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) - const result = await this.dagCbor.get(cid, { - signal: options?.signal, - onProgress: options?.onProgress - }) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) // return body as binary - const body = await this.helia.blockstore.get(cid) + const block = await this.helia.blockstore.get(cid) + let body: string | Uint8Array + + try { + body = dagCborToSafeJSON(block) + } catch { + body = block + } + const response = okResponse(body) - // return pre-parsed object with embedded CIDs as objects - response.json = async () => result - await this.setContentType(result, path, response) + response.headers.set('content-type', body instanceof Uint8Array ? 'application/octet-stream' : 'application/json') + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) return response } @@ -200,7 +178,6 @@ export class VerifiedFetch { signal: options?.signal, onProgress: options?.onProgress }) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: resolvedCID, path: '' })) this.log('got async iterator for %c/%s', cid, path) const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, { @@ -209,6 +186,8 @@ export class VerifiedFetch { const response = okResponse(stream) await this.setContentType(firstChunk, path, response) + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: resolvedCID, path: '' })) + return response } @@ -216,9 +195,10 @@ export class VerifiedFetch { this.log.trace('fetching %c/%s', cid, path) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) const result = await this.helia.blockstore.get(cid) - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - const response = okResponse(decode(result)) + const response = okResponse(raw.decode(result)) await this.setContentType(result, path, response) + + options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) return response } @@ -295,11 +275,11 @@ export class VerifiedFetch { } private readonly codecHandlers: Record = { - [dagJsonCode]: this.handleDagJson, - [dagPbCode]: this.handleDagPb, - [jsonCode]: this.handleJson, - [dagCborCode]: this.handleDagCbor, - [rawCode]: this.handleRaw, + [dagPb.code]: this.handleDagPb, + [dagJson.code]: this.handleJson, + [json.code]: this.handleJson, + [dagCbor.code]: this.handleDagCbor, + [raw.code]: this.handleRaw, [identity.code]: this.handleRaw } diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 938f9c35..6b455f13 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -4,6 +4,7 @@ import { dagJson } from '@helia/dag-json' import { type IPNS } from '@helia/ipns' import { json } from '@helia/json' import { unixfs, type UnixFS } from '@helia/unixfs' +import * as ipldDagCbor from '@ipld/dag-cbor' import { stop } from '@libp2p/interface' import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' @@ -221,6 +222,8 @@ describe('@helia/verifed-fetch', () => { expect(resp).to.be.ok() expect(resp.status).to.equal(200) expect(resp.statusText).to.equal('OK') + expect(resp.headers.get('content-type')).to.equal('application/json') + await expect(resp.json()).to.eventually.deep.equal(obj) }) @@ -233,14 +236,40 @@ describe('@helia/verifed-fetch', () => { const cid = await j.add(obj) const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + const data = await resp.json() expect(data).to.deep.equal({ hello: 'world', - link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + link: { + '/': 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' + } + }) + }) + + it('should return dag-json data with embedded bytes', async () => { + const obj = { + hello: 'world', + bytes: Uint8Array.from([0, 1, 2, 3, 4]) + } + const j = dagJson(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const data = await resp.json() + expect(data).to.deep.equal({ + hello: 'world', + bytes: { + '/': { + bytes: 'AAECAwQ' + } + } }) }) - it('should handle dag-cbor block', async () => { + it('should handle JSON-compliant dag-cbor block', async () => { const obj = { hello: 'world' } @@ -251,6 +280,7 @@ describe('@helia/verifed-fetch', () => { expect(resp).to.be.ok() expect(resp.status).to.equal(200) expect(resp.statusText).to.equal('OK') + expect(resp.headers.get('content-type')).to.equal('application/json') await expect(resp.json()).to.eventually.deep.equal(obj) }) @@ -263,13 +293,72 @@ describe('@helia/verifed-fetch', () => { const cid = await c.add(obj) const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + const data = await resp.json() expect(data).to.deep.equal({ hello: 'world', - link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + link: { + '/': 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' + } }) }) + it('should return dag-cbor data with embedded bytes', async () => { + const obj = { + hello: 'world', + bytes: Uint8Array.from([0, 1, 2, 3, 4]) + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const data = await resp.json() + expect(data).to.deep.equal({ + hello: 'world', + bytes: { + '/': { + bytes: 'AAECAwQ' + } + } + }) + }) + + it('should return dag-cbor with a small BigInt as application/json', async () => { + const obj = { + hello: 'world', + bigInt: 10n + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const data = await resp.json() + expect(data).to.deep.equal({ + hello: 'world', + bigInt: 10 + }) + }) + + it('should return dag-cbor with a large BigInt as application/octet-stream', async () => { + const obj = { + hello: 'world', + bigInt: BigInt(Number.MAX_SAFE_INTEGER) + 1n + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/octet-stream') + + const data = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(data).to.deep.equal(obj) + }) + it('should handle json block', async () => { const obj = { hello: 'world' From 0575113b8d68a7053cebe78c3617a7f4fb6ea56b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 14 Feb 2024 13:05:11 +0000 Subject: [PATCH 04/13] chore: add docs --- packages/verified-fetch/README.md | 160 +++++++++++++++++- packages/verified-fetch/package.json | 6 +- packages/verified-fetch/src/index.ts | 154 ++++++++++++++++- .../src/utils/dag-cbor-to-safe-json.ts | 24 +-- .../test/verified-fetch.spec.ts | 63 +++++-- 5 files changed, 365 insertions(+), 42 deletions(-) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index 0da153a6..40f976ee 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -126,7 +126,7 @@ const json = await resp.json() ### Custom content-type parsing -By default, `@helia/verified-fetch` sets the `Content-Type` header as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type. +By default, if the response can be parsed as JSON, `@helia/verified-fetch` sets the `Content-Type` header as `application/json`, otherwise it sets it as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type. If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type. @@ -140,14 +140,168 @@ import { fileTypeFromBuffer } from '@sgtpooki/file-type' const fetch = await createVerifiedFetch({ gateways: ['https://trustless-gateway.link'], - routers: ['http://delegated-ipfs.dev'], + routers: ['http://delegated-ipfs.dev'] +}, { contentTypeParser: async (bytes) => { // call to some magic-byte recognition library like magic-bytes, file-type, or your own custom byte recognition - return fileTypeFromBuffer(bytes)?.mime + const result = await fileTypeFromBuffer(bytes) + return result?.mime } }) ``` +### IPLD codec handling + +IPFS supports several data formats and `@helia/verified-fetch` attempts to abstract away some of the details. + +#### DAG-PB + +[DAG-PB](https://ipld.io/docs/codecs/known/dag-pb/) is the codec we are most likely to encounter, it is what [UnixFS](https://github.com/ipfs/specs/blob/main/UNIXFS.md) uses under the hood. + +##### Using the DAG-PB codec as a Blob + +```TypeScript +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://Qmfoo') +const blob = await res.blob() + +console.info(blob) // Blob { size: x, type: 'application/octet-stream' } +``` + +##### Using the DAG-PB codec as an ArrayBuffer + +```TypeScript +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://Qmfoo') +const buf = await res.arrayBuffer() + +console.info(buf) // ArrayBuffer { [Uint8Contents]: < ... >, byteLength: x } +``` + +##### Using the DAG-PB codec as a stream + +```TypeScript +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://Qmfoo') +const reader = res.body?.getReader() + +while (true) { + const next = await reader.read() + + if (next?.done === true) { + break + } + + if (next?.value != null) { + console.info(next.value) // Uint8Array(x) [ ... ] + } +} +``` + +##### Content-Type + +When fetching `DAG-PB` data, the content type will be set to `application/octet-stream` unless a custom content-type parser is configured. + +#### JSON + +The JSON codec is a very simple codec, a block parseable with this codec is a JSON string encoded into a `Uint8Array`. + +##### Using the JSON codec + +```TypeScript +import * as json from 'multiformats/codecs/json' + +const block = new TextEncoder().encode('{ "hello": "world" }') +const obj = json.decode(block) + +console.info(obj) // { hello: 'world' } +``` + +##### Content-Type + +When the `JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. + +### DAG-JSON + +[DAG-JSON](https://ipld.io/docs/codecs/known/dag-json/) expands on the `JSON` codec, adding the ability to contain [CID](https://docs.ipfs.tech/concepts/content-addressing/)s which act as links to other blocks, and byte arrays. + +`CID`s and byte arrays are represented using special object structures with a single `"/"` property. + +Using `DAG-JSON` has two important caveats: + +1. Your `JSON` structure cannot contain an object with only a `"/"` property, as it will be interpreted as a special type. +2. Since `JSON` has no technical limit on number sizes, `DAG-JSON` also allows numbers larger than `Number.MAX_SAFE_INTEGER`. JavaScript requires use of `BigInt`s to represent numbers larger than this, and `JSON.parse` does not support them, so precision will be lost. + +Otherwise this codec follows the same rules as the `JSON` codec. + +##### Using the DAG-JSON codec + +```TypeScript +import * as dagJson from '@ipld/dag-json' + +const block = new TextEncoder().encode(`{ + "hello": "world", + "cid": { + "/": "baeaaac3imvwgy3zao5xxe3de" + }, + "buf": { + "/": { + "bytes": "AAECAwQ" + } + } +}`) + +const obj = dagJson.decode(block) + +console.info(obj) +// { +// hello: 'world', +// cid: CID(baeaaac3imvwgy3zao5xxe3de), +// buf: Uint8Array(5) [ 0, 1, 2, 3, 4 ] +// } +``` + +##### Content-Type + +When the `DAG-JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. + +#### DAG-CBOR + +[DAG-CBOR](https://ipld.io/docs/codecs/known/dag-cbor/) uses the [Concise Binary Object Representation](https://cbor.io/) format for serialization instead of JSON. + +This supports more datatypes in a safer way than JSON and is smaller on the wire to boot so is usually preferable to JSON or DAG-JSON. + +##### Content-Type + +Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back. + +When a decoded block can be round-tripped to JSON, the `Content-Type` will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response. + +When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()`. + +##### Detecting JSON-safe DAG-CBOR + +```TypeScript +import { verifiedFetch } from '@helia/verified-fetch' +import * as dagCbor from '@ipld/dag-cbor' + +const res = await verifiedFetch('ipfs://bafyDagCborCID') +let obj + +if (res.headers.get('Content-Type') === 'application/json') { + // DAG-CBOR data can be safely decoded as JSON + obj = await res.json() +} else { + // response contains non-JSON friendly data types + obj = dagCbor.decode(new Uint8Array(await res.arrayBuffer())) +} + +console.info(obj) // ... +``` + ## Comparison to fetch This module attempts to act as similarly to the `fetch()` API as possible. diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 03bd4686..05af9be5 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -142,8 +142,6 @@ }, "dependencies": { "@helia/block-brokers": "^2.0.1", - "@helia/dag-cbor": "^3.0.0", - "@helia/dag-json": "^3.0.0", "@helia/http": "^1.0.1", "@helia/interface": "^4.0.0", "@helia/ipns": "^6.0.0", @@ -155,12 +153,15 @@ "@ipld/dag-pb": "^4.0.8", "@libp2p/interface": "^1.1.2", "@libp2p/peer-id": "^4.0.5", + "cborg": "^4.0.9", "hashlru": "^2.3.0", "ipfs-unixfs-exporter": "^13.5.0", "multiformats": "^13.0.1", "progress-events": "^1.0.0" }, "devDependencies": { + "@helia/dag-cbor": "^3.0.0", + "@helia/dag-json": "^3.0.0", "@helia/utils": "^0.0.1", "@libp2p/logger": "^4.0.5", "@libp2p/peer-id-factory": "^4.0.5", @@ -171,6 +172,7 @@ "datastore-core": "^9.2.8", "helia": "^4.0.1", "it-last": "^3.0.4", + "it-to-buffer": "^4.0.5", "magic-bytes.js": "^1.8.0", "sinon": "^17.0.1", "sinon-ts": "^2.0.0", diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 597d3a39..939d8618 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -114,7 +114,7 @@ * * ### Custom content-type parsing * - * By default, `@helia/verified-fetch` sets the `Content-Type` header as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type. + * By default, if the response can be parsed as JSON, `@helia/verified-fetch` sets the `Content-Type` header as `application/json`, otherwise it sets it as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type. * * If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type. * @@ -138,6 +138,158 @@ * }) * ``` * + * ### IPLD codec handling + * + * IPFS supports several data formats and `@helia/verified-fetch` attempts to abstract away some of the details. + * + * #### DAG-PB + * + * [DAG-PB](https://ipld.io/docs/codecs/known/dag-pb/) is the codec we are most likely to encounter, it is what [UnixFS](https://github.com/ipfs/specs/blob/main/UNIXFS.md) uses under the hood. + * + * ##### Using the DAG-PB codec as a Blob + * + * ```TypeScript + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://Qmfoo') + * const blob = await res.blob() + * + * console.info(blob) // Blob { size: x, type: 'application/octet-stream' } + * ``` + * + * ##### Using the DAG-PB codec as an ArrayBuffer + * + * ```TypeScript + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://Qmfoo') + * const buf = await res.arrayBuffer() + * + * console.info(buf) // ArrayBuffer { [Uint8Contents]: < ... >, byteLength: x } + * ``` + * + * ##### Using the DAG-PB codec as a stream + * + * ```TypeScript + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://Qmfoo') + * const reader = res.body?.getReader() + * + * while (true) { + * const next = await reader.read() + * + * if (next?.done === true) { + * break + * } + * + * if (next?.value != null) { + * console.info(next.value) // Uint8Array(x) [ ... ] + * } + * } + * ``` + * + * ##### Content-Type + * + * When fetching `DAG-PB` data, the content type will be set to `application/octet-stream` unless a custom content-type parser is configured. + * + * #### JSON + * + * The JSON codec is a very simple codec, a block parseable with this codec is a JSON string encoded into a `Uint8Array`. + * + * ##### Using the JSON codec + * + * ```TypeScript + * import * as json from 'multiformats/codecs/json' + * + * const block = new TextEncoder().encode('{ "hello": "world" }') + * const obj = json.decode(block) + * + * console.info(obj) // { hello: 'world' } + * ``` + * + * ##### Content-Type + * + * When the `JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. + * + * ### DAG-JSON + * + * [DAG-JSON](https://ipld.io/docs/codecs/known/dag-json/) expands on the `JSON` codec, adding the ability to contain [CID](https://docs.ipfs.tech/concepts/content-addressing/)s which act as links to other blocks, and byte arrays. + * + * `CID`s and byte arrays are represented using special object structures with a single `"/"` property. + * + * Using `DAG-JSON` has two important caveats: + * + * 1. Your `JSON` structure cannot contain an object with only a `"/"` property, as it will be interpreted as a special type. + * 2. Since `JSON` has no technical limit on number sizes, `DAG-JSON` also allows numbers larger than `Number.MAX_SAFE_INTEGER`. JavaScript requires use of `BigInt`s to represent numbers larger than this, and `JSON.parse` does not support them, so precision will be lost. + * + * Otherwise this codec follows the same rules as the `JSON` codec. + * + * ##### Using the DAG-JSON codec + * + * ```TypeScript + * import * as dagJson from '@ipld/dag-json' + * + * const block = new TextEncoder().encode(`{ + * "hello": "world", + * "cid": { + * "/": "baeaaac3imvwgy3zao5xxe3de" + * }, + * "buf": { + * "/": { + * "bytes": "AAECAwQ" + * } + * } + * }`) + * + * const obj = dagJson.decode(block) + * + * console.info(obj) + * // { + * // hello: 'world', + * // cid: CID(baeaaac3imvwgy3zao5xxe3de), + * // buf: Uint8Array(5) [ 0, 1, 2, 3, 4 ] + * // } + * ``` + * + * ##### Content-Type + * + * When the `DAG-JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. + * + * #### DAG-CBOR + * + * [DAG-CBOR](https://ipld.io/docs/codecs/known/dag-cbor/) uses the [Concise Binary Object Representation](https://cbor.io/) format for serialization instead of JSON. + * + * This supports more datatypes in a safer way than JSON and is smaller on the wire to boot so is usually preferable to JSON or DAG-JSON. + * + * ##### Content-Type + * + * Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back. + * + * When a decoded block can be round-tripped to JSON, the `Content-Type` will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response. + * + * When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()`. + * + * ##### Detecting JSON-safe DAG-CBOR + * + * ```TypeScript + * import { verifiedFetch } from '@helia/verified-fetch' + * import * as dagCbor from '@ipld/dag-cbor' + * + * const res = await verifiedFetch('ipfs://bafyDagCborCID') + * let obj + * + * if (res.headers.get('Content-Type') === 'application/json') { + * // DAG-CBOR data can be safely decoded as JSON + * obj = await res.json() + * } else { + * // response contains non-JSON friendly data types + * obj = dagCbor.decode(new Uint8Array(await res.arrayBuffer())) + * } + * + * console.info(obj) // ... + * ``` + * * ## Comparison to fetch * * This module attempts to act as similarly to the `fetch()` API as possible. diff --git a/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts b/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts index a6588809..0c52bdcc 100644 --- a/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts +++ b/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts @@ -1,5 +1,5 @@ -import * as dagJson from '@ipld/dag-json' import { decode } from 'cborg' +import { encode } from 'cborg/json' import { CID } from 'multiformats/cid' import type { TagDecoder } from 'cborg' @@ -18,20 +18,6 @@ function cidDecoder (bytes: Uint8Array): CID { * Take a `DAG-CBOR` encoded `Uint8Array`, deserialize it as an object and * re-serialize it in a form that can be passed to `JSON.serialize` and then * `JSON.parse` without losing any data. - * - * This is not as simple as it sounds because `DAG-CBOR` will return `BigInt`s - * for numbers with a value greater than `Number.MAX_SAFE_INTEGER` - `BigInt`s - * are not part of JSON so we disable support for them which will cause cborg - * to throw. - * - * `DAG-JSON` uses `Number`s exclusively even though JavaScript cannot safely - * decode them once they become greater than `Number.MAX_SAFE_INTEGER` so we - * doubly need to throw when they are encountered which will set the response - * type as `application/octet-stream` and the user can use `@ipld/dag-cbor` to - * decode the values in a safe way. - * - * `CID`s are re-encoded as `{ "/": "QmFoo" }` and `Uint8Array`s to - * `{ "/": { "bytes": "base64EncodedBytes" }}` as per the `DAG-JSON` spec. */ export function dagCborToSafeJSON (buf: Uint8Array): string { const tags: TagDecoder[] = [] @@ -48,11 +34,11 @@ export function dagCborToSafeJSON (buf: Uint8Array): string { tags, // this is different to `DAG-CBOR` - the reason we disallow BigInts is - // because we are about to re-encode to `DAG-JSON` which allows larger - // numbers than the JS Number type supports so we cause cborg to throw in - // order to prompt the user to decode using a method that preserves data. + // because we are about to re-encode to `JSON` which does not support + // BigInts. Blocks containing large numbers should be deserialized using a + // cbor decoder instead allowBigInt: false }) - return new TextDecoder().decode(dagJson.encode(obj)) + return new TextDecoder().decode(encode(obj)) } diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 6b455f13..3f89aa56 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -5,10 +5,12 @@ import { type IPNS } from '@helia/ipns' import { json } from '@helia/json' import { unixfs, type UnixFS } from '@helia/unixfs' import * as ipldDagCbor from '@ipld/dag-cbor' +import * as ipldDagJson from '@ipld/dag-json' import { stop } from '@libp2p/interface' import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' import last from 'it-last' +import toBuffer from 'it-to-buffer' import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' @@ -187,6 +189,31 @@ describe('@helia/verifed-fetch', () => { expect(new Uint8Array(data)).to.equalBytes(finalRootFileContent) }) + it('should allow use as a stream', async () => { + const content = new Uint8Array([0x01, 0x02, 0x03]) + + const fs = unixfs(helia) + const cid = await fs.addBytes(content) + + const res = await verifiedFetch.fetch(cid) + const reader = res.body?.getReader() + const output: Uint8Array[] = [] + + while (true) { + const next = await reader?.read() + + if (next?.done === true) { + break + } + + if (next?.value != null) { + output.push(next.value) + } + } + + expect(toBuffer(output)).to.equalBytes(content) + }) + it('should return 501 if index file is not found', async () => { const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) @@ -293,15 +320,10 @@ describe('@helia/verifed-fetch', () => { const cid = await c.add(obj) const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') + expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - const data = await resp.json() - expect(data).to.deep.equal({ - hello: 'world', - link: { - '/': 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' - } - }) + const data = await ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(data).to.deep.equal(obj) }) it('should return dag-cbor data with embedded bytes', async () => { @@ -312,18 +334,25 @@ describe('@helia/verifed-fetch', () => { const c = dagCbor(helia) const cid = await c.add(obj) + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/octet-stream') + + const data = await ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + expect(data).to.deep.equal(obj) + }) + + it('should allow parsing dag-cbor object array buffer as dag-json', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/json') - const data = await resp.json() - expect(data).to.deep.equal({ - hello: 'world', - bytes: { - '/': { - bytes: 'AAECAwQ' - } - } - }) + const data = ipldDagJson.decode(new Uint8Array(await resp.arrayBuffer())) + expect(data).to.deep.equal(obj) }) it('should return dag-cbor with a small BigInt as application/json', async () => { From e6bf70ed16eab81cc8cb102de1bf7d347b13d523 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 14 Feb 2024 13:13:49 +0000 Subject: [PATCH 05/13] chore: more docs --- packages/verified-fetch/README.md | 28 ++++++++++++++++++++++++++++ packages/verified-fetch/src/index.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index 40f976ee..e3f0d988 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -268,6 +268,34 @@ console.info(obj) When the `DAG-JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. +`DAG-JSON` data can be parsed from the response by using the `.json()` function, which will return `CID`s/byte arrays as plain `{ "/": ... }` objects: + +```TypeScript +import { verifiedFetch } from '@helia/verified-fetch' +import * as dagJson from '@ipld/dag-json' + +const res = await verifiedFetch('ipfs://bafyDAGJSON') + +// either: +const obj = await res.json() +console.info(obj.cid) // { "/": "baeaaac3imvwgy3zao5xxe3de" } +console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } } +``` + +Alternatively or it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get CID`objects and`Uint8Array\`s: + +```TypeScript +import { verifiedFetch } from '@helia/verified-fetch' +import * as dagJson from '@ipld/dag-json' + +const res = await verifiedFetch('ipfs://bafyDAGJSON') + +// or: +const obj = dagJson.decode(new Uint8Array(await res.arrayBuffer())) +console.info(obj.cid) // CID(baeaaac3imvwgy3zao5xxe3de) +console.info(obj.buf) // Uint8Array(5) [ 0, 1, 2, 3, 4 ] +``` + #### DAG-CBOR [DAG-CBOR](https://ipld.io/docs/codecs/known/dag-cbor/) uses the [Concise Binary Object Representation](https://cbor.io/) format for serialization instead of JSON. diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 939d8618..36e204d5 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -256,6 +256,34 @@ * * When the `DAG-JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. * + * `DAG-JSON` data can be parsed from the response by using the `.json()` function, which will return `CID`s/byte arrays as plain `{ "/": ... }` objects: + * + * ```TypeScript + * import { verifiedFetch } from '@helia/verified-fetch' + * import * as dagJson from '@ipld/dag-json' + * + * const res = await verifiedFetch('ipfs://bafyDAGJSON') + * + * // either: + * const obj = await res.json() + * console.info(obj.cid) // { "/": "baeaaac3imvwgy3zao5xxe3de" } + * console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } } + * ``` + * + * Alternatively or it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get CID` objects and `Uint8Array`s: + * + *```TypeScript + * import { verifiedFetch } from '@helia/verified-fetch' + * import * as dagJson from '@ipld/dag-json' + * + * const res = await verifiedFetch('ipfs://bafyDAGJSON') + * + * // or: + * const obj = dagJson.decode(new Uint8Array(await res.arrayBuffer())) + * console.info(obj.cid) // CID(baeaaac3imvwgy3zao5xxe3de) + * console.info(obj.buf) // Uint8Array(5) [ 0, 1, 2, 3, 4 ] + * ``` + * * #### DAG-CBOR * * [DAG-CBOR](https://ipld.io/docs/codecs/known/dag-cbor/) uses the [Concise Binary Object Representation](https://cbor.io/) format for serialization instead of JSON. From 6e67ab7606493c5e0bef4621f34652dcbd9eb380 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 14 Feb 2024 13:15:01 +0000 Subject: [PATCH 06/13] chore: docs again --- packages/verified-fetch/README.md | 2 +- packages/verified-fetch/src/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index e3f0d988..c207b9fa 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -282,7 +282,7 @@ console.info(obj.cid) // { "/": "baeaaac3imvwgy3zao5xxe3de" } console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } } ``` -Alternatively or it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get CID`objects and`Uint8Array\`s: +Alternatively or it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s: ```TypeScript import { verifiedFetch } from '@helia/verified-fetch' diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 36e204d5..4a517699 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -270,7 +270,7 @@ * console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } } * ``` * - * Alternatively or it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get CID` objects and `Uint8Array`s: + * Alternatively or it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s: * *```TypeScript * import { verifiedFetch } from '@helia/verified-fetch' From 58e576def9b099bbb7ed6ee8e6fe46f22e61b0eb Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 14 Feb 2024 13:16:44 +0000 Subject: [PATCH 07/13] chore: docs --- packages/verified-fetch/README.md | 2 ++ packages/verified-fetch/src/index.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index c207b9fa..e0c0535f 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -312,6 +312,8 @@ When it cannot, the `Content-Type` will be `application/octet-stream` - in this ##### Detecting JSON-safe DAG-CBOR +If the `Content-Type` header of the response is `application/json`, the `.json()` method may be used to access the response body in object form, otherwise the `.arrayBuffer()` method must be used to decode the raw bytes using the `@ipld/dag-cbor` module. + ```TypeScript import { verifiedFetch } from '@helia/verified-fetch' import * as dagCbor from '@ipld/dag-cbor' diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 4a517699..66ec8b79 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -300,6 +300,8 @@ * * ##### Detecting JSON-safe DAG-CBOR * + * If the `Content-Type` header of the response is `application/json`, the `.json()` method may be used to access the response body in object form, otherwise the `.arrayBuffer()` method must be used to decode the raw bytes using the `@ipld/dag-cbor` module. + * * ```TypeScript * import { verifiedFetch } from '@helia/verified-fetch' * import * as dagCbor from '@ipld/dag-cbor' From 77a31c4dfa2a8cfdd58cfbfd16d80cac27410247 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 14 Feb 2024 15:02:27 +0000 Subject: [PATCH 08/13] chore: skip json decode --- packages/verified-fetch/package.json | 2 +- packages/verified-fetch/src/verified-fetch.ts | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 05af9be5..f4de2f8c 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -145,7 +145,6 @@ "@helia/http": "^1.0.1", "@helia/interface": "^4.0.0", "@helia/ipns": "^6.0.0", - "@helia/json": "^3.0.0", "@helia/routers": "^1.0.0", "@helia/unixfs": "^3.0.0", "@ipld/dag-cbor": "^9.1.0", @@ -162,6 +161,7 @@ "devDependencies": { "@helia/dag-cbor": "^3.0.0", "@helia/dag-json": "^3.0.0", + "@helia/json": "^3.0.0", "@helia/utils": "^0.0.1", "@libp2p/logger": "^4.0.5", "@libp2p/peer-id-factory": "^4.0.5", diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index fd901dd0..9119238c 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -1,6 +1,5 @@ import { ipns as heliaIpns, type IPNS } from '@helia/ipns' import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' -import { json as heliaJson, type JSON } from '@helia/json' import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' import { code as dagCborCode } from '@ipld/dag-cbor' import { code as dagJsonCode } from '@ipld/dag-json' @@ -23,7 +22,6 @@ interface VerifiedFetchComponents { helia: Helia ipns?: IPNS unixfs?: HeliaUnixFs - json?: JSON pathWalker?: PathWalkerFn } @@ -78,12 +76,11 @@ export class VerifiedFetch { private readonly helia: Helia private readonly ipns: IPNS private readonly unixfs: HeliaUnixFs - private readonly json: JSON private readonly pathWalker: PathWalkerFn private readonly log: Logger private readonly contentTypeParser: ContentTypeParser | undefined - constructor ({ helia, ipns, unixfs, json, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) { + constructor ({ helia, ipns, unixfs, pathWalker }: VerifiedFetchComponents, init?: VerifiedFetchInit) { this.helia = helia this.log = helia.logger.forComponent('helia:verified-fetch') this.ipns = ipns ?? heliaIpns(helia, { @@ -93,7 +90,6 @@ export class VerifiedFetch { ] }) this.unixfs = unixfs ?? heliaUnixFs(helia) - this.json = json ?? heliaJson(helia) this.pathWalker = pathWalker ?? walkPath this.contentTypeParser = init?.contentTypeParser this.log.trace('created VerifiedFetch instance') @@ -116,11 +112,11 @@ export class VerifiedFetch { private async handleJson ({ cid, path, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { cid, path })) - const result: Record = await this.json.get(cid, { + const result = await this.helia.blockstore.get(cid, { signal: options?.signal, onProgress: options?.onProgress }) - const response = okResponse(JSON.stringify(result)) + const response = okResponse(result) response.headers.set('content-type', 'application/json') options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) return response From 6c32f5c0742a5122621aea68b75264e67c88f789 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 14 Feb 2024 15:04:51 +0000 Subject: [PATCH 09/13] chore: log error --- packages/verified-fetch/src/verified-fetch.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 9119238c..9c256c9a 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -131,7 +131,8 @@ export class VerifiedFetch { try { body = dagCborToSafeJSON(block) - } catch { + } catch (err) { + this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err) body = block } From 22376fffbc3b5496f980116b0eae624f4877732b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 14 Feb 2024 15:33:16 +0000 Subject: [PATCH 10/13] chore: add CID-round trip tests --- .../test/verified-fetch.spec.ts | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 3f89aa56..fd4fb553 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -12,6 +12,7 @@ import { expect } from 'aegir/chai' import last from 'it-last' import toBuffer from 'it-to-buffer' import { CID } from 'multiformats/cid' +import * as ipldJson from 'multiformats/codecs/json' import * as raw from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import { sha256 } from 'multiformats/hashes/sha2' @@ -238,6 +239,34 @@ describe('@helia/verifed-fetch', () => { expect(resp.statusText).to.equal('Not Implemented') }) + it('can round trip json via .json()', async () => { + const obj = { + hello: 'world' + } + const j = json(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const output = await resp.json() + await expect(j.add(output)).to.eventually.deep.equal(cid) + }) + + it('can round trip json via .arrayBuffer()', async () => { + const obj = { + hello: 'world' + } + const j = json(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const output = ipldJson.decode(new Uint8Array(await resp.arrayBuffer())) + await expect(j.add(output)).to.eventually.deep.equal(cid) + }) + it('should handle dag-json block', async () => { const obj = { hello: 'world' @@ -296,6 +325,40 @@ describe('@helia/verifed-fetch', () => { }) }) + it('can round trip dag-json via .json()', async () => { + const obj = { + hello: 'world', + // n.b. cannot round-trip larger than Number.MAX_SAFE_INTEGER because + // parsing DAG-JSON as using JSON.parse loses precision + bigInt: 10n, + link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + } + const j = dagJson(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const output = await resp.json() + await expect(j.add(output)).to.eventually.deep.equal(cid) + }) + + it('can round trip dag-json via .arrayBuffer()', async () => { + const obj = { + hello: 'world', + bigInt: BigInt(Number.MAX_SAFE_INTEGER) + 1n, + link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + } + const j = dagJson(helia) + const cid = await j.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const output = ipldDagJson.decode(new Uint8Array(await resp.arrayBuffer())) + await expect(j.add(output)).to.eventually.deep.equal(cid) + }) + it('should handle JSON-compliant dag-cbor block', async () => { const obj = { hello: 'world' @@ -388,6 +451,53 @@ describe('@helia/verifed-fetch', () => { expect(data).to.deep.equal(obj) }) + it('can round trip JSON-compliant dag-cbor via .json()', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const output = await resp.json() + await expect(c.add(output)).to.eventually.deep.equal(cid) + }) + + // N.b. this is not possible because the incoming block is turned into JSON + // and returned as the response body, so `.arrayBufer()` returns a string + // encoded into a Uint8Array which we can't parse as CBOR + it.skip('can round trip JSON-compliant dag-cbor via .arrayBuffer()', async () => { + const obj = { + hello: 'world' + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/json') + + const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + await expect(c.add(output)).to.eventually.deep.equal(cid) + }) + + it('can round trip dag-cbor via .arrayBuffer()', async () => { + const obj = { + hello: 'world', + bigInt: BigInt(Number.MAX_SAFE_INTEGER) + 1n, + link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') + } + const c = dagCbor(helia) + const cid = await c.add(obj) + + const resp = await verifiedFetch.fetch(cid) + expect(resp.headers.get('content-type')).to.equal('application/octet-stream') + + const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + await expect(c.add(output)).to.eventually.deep.equal(cid) + }) + it('should handle json block', async () => { const obj = { hello: 'world' From d69c2c449d2fbd51e4df7714759761500365cd68 Mon Sep 17 00:00:00 2001 From: Daniel Norman <1992255+2color@users.noreply.github.com> Date: Thu, 15 Feb 2024 16:35:51 +0100 Subject: [PATCH 11/13] docs: refine verified fetch docs (#436) Co-authored-by: Daniel N <2color@users.noreply.github.com> --- packages/verified-fetch/src/index.ts | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 66ec8b79..3fd0e634 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -3,7 +3,7 @@ * * `@helia/verified-fetch` provides a [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)-like API for retrieving content from the [IPFS](https://ipfs.tech/) network. * - * All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. + * All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. By default, CIDs are retrieved over HTTP from [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/). * * This is a marked improvement over `fetch` which offers no such protections and is vulnerable to all sorts of attacks like [Content Spoofing](https://owasp.org/www-community/attacks/Content_Spoofing), [DNS Hijacking](https://en.wikipedia.org/wiki/DNS_hijacking), etc. * @@ -33,7 +33,7 @@ * import { verifiedFetch } from '@helia/verified-fetch' * import { CID } from 'multiformats/cid' * - * const cid = CID.parse('bafyFoo') // some image file + * const cid = CID.parse('bafyFoo') // some json file * const response = await verifiedFetch(cid) * const json = await response.json() * ``` @@ -140,7 +140,7 @@ * * ### IPLD codec handling * - * IPFS supports several data formats and `@helia/verified-fetch` attempts to abstract away some of the details. + * IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption. * * #### DAG-PB * @@ -148,7 +148,7 @@ * * ##### Using the DAG-PB codec as a Blob * - * ```TypeScript + * ```typescript * import { verifiedFetch } from '@helia/verified-fetch' * * const res = await verifiedFetch('ipfs://Qmfoo') @@ -159,7 +159,7 @@ * * ##### Using the DAG-PB codec as an ArrayBuffer * - * ```TypeScript + * ```typescript * import { verifiedFetch } from '@helia/verified-fetch' * * const res = await verifiedFetch('ipfs://Qmfoo') @@ -170,7 +170,7 @@ * * ##### Using the DAG-PB codec as a stream * - * ```TypeScript + * ```typescript * import { verifiedFetch } from '@helia/verified-fetch' * * const res = await verifiedFetch('ipfs://Qmfoo') @@ -199,7 +199,7 @@ * * ##### Using the JSON codec * - * ```TypeScript + * ```typescript * import * as json from 'multiformats/codecs/json' * * const block = new TextEncoder().encode('{ "hello": "world" }') @@ -227,7 +227,7 @@ * * ##### Using the DAG-JSON codec * - * ```TypeScript + * ```typescript * import * as dagJson from '@ipld/dag-json' * * const block = new TextEncoder().encode(`{ @@ -254,11 +254,11 @@ * * ##### Content-Type * - * When the `DAG-JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. + * When the `DAG-JSON` codec is encountered in the requested CID, the `Content-Type` header of the response will be set to `application/json`. * * `DAG-JSON` data can be parsed from the response by using the `.json()` function, which will return `CID`s/byte arrays as plain `{ "/": ... }` objects: * - * ```TypeScript + * ```typescript * import { verifiedFetch } from '@helia/verified-fetch' * import * as dagJson from '@ipld/dag-json' * @@ -270,9 +270,9 @@ * console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } } * ``` * - * Alternatively or it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s: + * Alternatively, it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s: * - *```TypeScript + *```typescript * import { verifiedFetch } from '@helia/verified-fetch' * import * as dagJson from '@ipld/dag-json' * @@ -292,7 +292,7 @@ * * ##### Content-Type * - * Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back. + * Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back into the same binary form. * * When a decoded block can be round-tripped to JSON, the `Content-Type` will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response. * @@ -302,7 +302,7 @@ * * If the `Content-Type` header of the response is `application/json`, the `.json()` method may be used to access the response body in object form, otherwise the `.arrayBuffer()` method must be used to decode the raw bytes using the `@ipld/dag-cbor` module. * - * ```TypeScript + * ```typescript * import { verifiedFetch } from '@helia/verified-fetch' * import * as dagCbor from '@ipld/dag-cbor' * @@ -337,7 +337,7 @@ * 2. IPNS protocol: `ipns://` & `ipns://` & `ipns://` * 3. CID instances: An actual CID instance `CID.parse('bafy...')` * - * As well as support for pathing & params for item 1 & 2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453). + * As well as support for pathing & params for items 1 & 2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453). * * If you pass a CID instance, it assumes you want the content for that specific CID only, and does not support pathing or params for that CID. * From 327713102b1b100fbd1ebdb83b48761841efd453 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 16 Feb 2024 15:57:03 +0000 Subject: [PATCH 12/13] chore: apply suggestions from code review Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> --- packages/verified-fetch/README.md | 4 ++-- packages/verified-fetch/test/verified-fetch.spec.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index e0c0535f..b8b27468 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -306,9 +306,9 @@ This supports more datatypes in a safer way than JSON and is smaller on the wire Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back. -When a decoded block can be round-tripped to JSON, the `Content-Type` will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response. +When a decoded block can be round-tripped to JSON, the response body will be a JSON string and the `Content-Type` header will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response and the `.arrayBuffer()` method will return the JSON string as a byte array - note that this will not be parsable by the `@ipld/dag-cbor` module. -When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()`. +When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()` and the `.json()` method cannot be used since the response body will be unparsed `CBOR` bytes. ##### Detecting JSON-safe DAG-CBOR diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index fd4fb553..c5e7456c 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -466,7 +466,7 @@ describe('@helia/verifed-fetch', () => { }) // N.b. this is not possible because the incoming block is turned into JSON - // and returned as the response body, so `.arrayBufer()` returns a string + // and returned as the response body, so `.arrayBuffer()` returns a string // encoded into a Uint8Array which we can't parse as CBOR it.skip('can round trip JSON-compliant dag-cbor via .arrayBuffer()', async () => { const obj = { From 7884c27d19e153c1af8b275b23092a5ddb2fd6d4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 16 Feb 2024 16:04:21 +0000 Subject: [PATCH 13/13] chore: remove extra uint8array creation --- packages/verified-fetch/README.md | 53 ++++++++++++------- packages/verified-fetch/package.json | 8 +-- packages/verified-fetch/src/index.ts | 4 +- .../test/verified-fetch.spec.ts | 16 +++--- 4 files changed, 48 insertions(+), 33 deletions(-) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index b8b27468..b355422c 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -13,9 +13,24 @@ # About + + `@helia/verified-fetch` provides a [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)-like API for retrieving content from the [IPFS](https://ipfs.tech/) network. -All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. +All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. By default, CIDs are retrieved over HTTP from [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/). This is a marked improvement over `fetch` which offers no such protections and is vulnerable to all sorts of attacks like [Content Spoofing](https://owasp.org/www-community/attacks/Content_Spoofing), [DNS Hijacking](https://en.wikipedia.org/wiki/DNS_hijacking), etc. @@ -45,7 +60,7 @@ const json = await resp.json() import { verifiedFetch } from '@helia/verified-fetch' import { CID } from 'multiformats/cid' -const cid = CID.parse('bafyFoo') // some image file +const cid = CID.parse('bafyFoo') // some json file const response = await verifiedFetch(cid) const json = await response.json() ``` @@ -152,7 +167,7 @@ const fetch = await createVerifiedFetch({ ### IPLD codec handling -IPFS supports several data formats and `@helia/verified-fetch` attempts to abstract away some of the details. +IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption. #### DAG-PB @@ -160,7 +175,7 @@ IPFS supports several data formats and `@helia/verified-fetch` attempts to abstr ##### Using the DAG-PB codec as a Blob -```TypeScript +```typescript import { verifiedFetch } from '@helia/verified-fetch' const res = await verifiedFetch('ipfs://Qmfoo') @@ -171,7 +186,7 @@ console.info(blob) // Blob { size: x, type: 'application/octet-stream' } ##### Using the DAG-PB codec as an ArrayBuffer -```TypeScript +```typescript import { verifiedFetch } from '@helia/verified-fetch' const res = await verifiedFetch('ipfs://Qmfoo') @@ -182,7 +197,7 @@ console.info(buf) // ArrayBuffer { [Uint8Contents]: < ... >, byteLength: x } ##### Using the DAG-PB codec as a stream -```TypeScript +```typescript import { verifiedFetch } from '@helia/verified-fetch' const res = await verifiedFetch('ipfs://Qmfoo') @@ -211,7 +226,7 @@ The JSON codec is a very simple codec, a block parseable with this codec is a JS ##### Using the JSON codec -```TypeScript +```typescript import * as json from 'multiformats/codecs/json' const block = new TextEncoder().encode('{ "hello": "world" }') @@ -239,7 +254,7 @@ Otherwise this codec follows the same rules as the `JSON` codec. ##### Using the DAG-JSON codec -```TypeScript +```typescript import * as dagJson from '@ipld/dag-json' const block = new TextEncoder().encode(`{ @@ -266,11 +281,11 @@ console.info(obj) ##### Content-Type -When the `DAG-JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. +When the `DAG-JSON` codec is encountered in the requested CID, the `Content-Type` header of the response will be set to `application/json`. `DAG-JSON` data can be parsed from the response by using the `.json()` function, which will return `CID`s/byte arrays as plain `{ "/": ... }` objects: -```TypeScript +```typescript import { verifiedFetch } from '@helia/verified-fetch' import * as dagJson from '@ipld/dag-json' @@ -282,16 +297,16 @@ console.info(obj.cid) // { "/": "baeaaac3imvwgy3zao5xxe3de" } console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } } ``` -Alternatively or it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s: +Alternatively, it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s: -```TypeScript +```typescript import { verifiedFetch } from '@helia/verified-fetch' import * as dagJson from '@ipld/dag-json' const res = await verifiedFetch('ipfs://bafyDAGJSON') // or: -const obj = dagJson.decode(new Uint8Array(await res.arrayBuffer())) +const obj = dagJson.decode(await res.arrayBuffer()) console.info(obj.cid) // CID(baeaaac3imvwgy3zao5xxe3de) console.info(obj.buf) // Uint8Array(5) [ 0, 1, 2, 3, 4 ] ``` @@ -304,17 +319,17 @@ This supports more datatypes in a safer way than JSON and is smaller on the wire ##### Content-Type -Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back. +Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back into the same binary form. -When a decoded block can be round-tripped to JSON, the response body will be a JSON string and the `Content-Type` header will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response and the `.arrayBuffer()` method will return the JSON string as a byte array - note that this will not be parsable by the `@ipld/dag-cbor` module. +When a decoded block can be round-tripped to JSON, the `Content-Type` will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response. -When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()` and the `.json()` method cannot be used since the response body will be unparsed `CBOR` bytes. +When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()`. ##### Detecting JSON-safe DAG-CBOR If the `Content-Type` header of the response is `application/json`, the `.json()` method may be used to access the response body in object form, otherwise the `.arrayBuffer()` method must be used to decode the raw bytes using the `@ipld/dag-cbor` module. -```TypeScript +```typescript import { verifiedFetch } from '@helia/verified-fetch' import * as dagCbor from '@ipld/dag-cbor' @@ -326,7 +341,7 @@ if (res.headers.get('Content-Type') === 'application/json') { obj = await res.json() } else { // response contains non-JSON friendly data types - obj = dagCbor.decode(new Uint8Array(await res.arrayBuffer())) + obj = dagCbor.decode(await res.arrayBuffer()) } console.info(obj) // ... @@ -349,7 +364,7 @@ This library supports the following methods of fetching web3 content from IPFS: 2. IPNS protocol: `ipns://` & `ipns://` & `ipns://` 3. CID instances: An actual CID instance `CID.parse('bafy...')` -As well as support for pathing & params for item 1 & 2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453). +As well as support for pathing & params for items 1 & 2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453). If you pass a CID instance, it assumes you want the content for that specific CID only, and does not support pathing or params for that CID. diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index f4de2f8c..c710cf61 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -147,15 +147,15 @@ "@helia/ipns": "^6.0.0", "@helia/routers": "^1.0.0", "@helia/unixfs": "^3.0.0", - "@ipld/dag-cbor": "^9.1.0", - "@ipld/dag-json": "^10.1.7", - "@ipld/dag-pb": "^4.0.8", + "@ipld/dag-cbor": "^9.2.0", + "@ipld/dag-json": "^10.2.0", + "@ipld/dag-pb": "^4.1.0", "@libp2p/interface": "^1.1.2", "@libp2p/peer-id": "^4.0.5", "cborg": "^4.0.9", "hashlru": "^2.3.0", "ipfs-unixfs-exporter": "^13.5.0", - "multiformats": "^13.0.1", + "multiformats": "^13.1.0", "progress-events": "^1.0.0" }, "devDependencies": { diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 3fd0e634..786f2573 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -279,7 +279,7 @@ * const res = await verifiedFetch('ipfs://bafyDAGJSON') * * // or: - * const obj = dagJson.decode(new Uint8Array(await res.arrayBuffer())) + * const obj = dagJson.decode(await res.arrayBuffer()) * console.info(obj.cid) // CID(baeaaac3imvwgy3zao5xxe3de) * console.info(obj.buf) // Uint8Array(5) [ 0, 1, 2, 3, 4 ] * ``` @@ -314,7 +314,7 @@ * obj = await res.json() * } else { * // response contains non-JSON friendly data types - * obj = dagCbor.decode(new Uint8Array(await res.arrayBuffer())) + * obj = dagCbor.decode(await res.arrayBuffer()) * } * * console.info(obj) // ... diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index c5e7456c..d495c031 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -263,7 +263,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/json') - const output = ipldJson.decode(new Uint8Array(await resp.arrayBuffer())) + const output = ipldJson.decode(await resp.arrayBuffer()) await expect(j.add(output)).to.eventually.deep.equal(cid) }) @@ -355,7 +355,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/json') - const output = ipldDagJson.decode(new Uint8Array(await resp.arrayBuffer())) + const output = ipldDagJson.decode(await resp.arrayBuffer()) await expect(j.add(output)).to.eventually.deep.equal(cid) }) @@ -385,7 +385,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - const data = await ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + const data = await ipldDagCbor.decode(await resp.arrayBuffer()) expect(data).to.deep.equal(obj) }) @@ -400,7 +400,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - const data = await ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + const data = await ipldDagCbor.decode(await resp.arrayBuffer()) expect(data).to.deep.equal(obj) }) @@ -414,7 +414,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/json') - const data = ipldDagJson.decode(new Uint8Array(await resp.arrayBuffer())) + const data = ipldDagJson.decode(await resp.arrayBuffer()) expect(data).to.deep.equal(obj) }) @@ -447,7 +447,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - const data = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + const data = ipldDagCbor.decode(await resp.arrayBuffer()) expect(data).to.deep.equal(obj) }) @@ -478,7 +478,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/json') - const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + const output = ipldDagCbor.decode(await resp.arrayBuffer()) await expect(c.add(output)).to.eventually.deep.equal(cid) }) @@ -494,7 +494,7 @@ describe('@helia/verifed-fetch', () => { const resp = await verifiedFetch.fetch(cid) expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) + const output = ipldDagCbor.decode(await resp.arrayBuffer()) await expect(c.add(output)).to.eventually.deep.equal(cid) })