From 546bf6685a414e0cf470894704ec7ef7a5654e9c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 19 Jan 2024 11:59:08 +0100 Subject: [PATCH 01/27] feat: add block session support to @helia/interface There are no implementations yet but the usage pattern will be something like: ```javascript // unixfs cat command export async function * cat (cid: CID, blockstore: Blocks, options: Partial = {}): AsyncIterable { // create a session for the CID if support is available const blocks = await (blockstore.createSession != null ? blockstore.createSession(cid) : blockstore) const opts: CatOptions = mergeOptions(defaultOptions, options) // resolve and export using the session, if created, otherwise fall back to regular blockstore access const resolved = await resolve(cid, opts.path, blocks, opts) const result = await exporter(resolved.cid, blocks, opts) if (result.type !== 'file' && result.type !== 'raw') { throw new NotAFileError() } if (result.content == null) { throw new NoContentError() } yield * result.content(opts) } ``` --- packages/block-brokers/src/bitswap.ts | 8 +- .../src/trustless-gateway/broker.ts | 4 +- .../src/trustless-gateway/index.ts | 6 +- .../test/trustless-gateway.spec.ts | 12 +- packages/interface/src/blocks.ts | 32 +++-- packages/utils/src/storage.ts | 22 +++- packages/utils/src/utils/networked-storage.ts | 122 +++++++++++------- packages/utils/test/block-broker.spec.ts | 4 +- packages/utils/test/storage.spec.ts | 4 +- .../test/utils/networked-storage.spec.ts | 4 +- 10 files changed, 136 insertions(+), 82 deletions(-) diff --git a/packages/block-brokers/src/bitswap.ts b/packages/block-brokers/src/bitswap.ts index 072594fc..f764e233 100644 --- a/packages/block-brokers/src/bitswap.ts +++ b/packages/block-brokers/src/bitswap.ts @@ -1,5 +1,5 @@ import { createBitswap } from 'ipfs-bitswap' -import type { BlockAnnouncer, BlockBroker, BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks' +import type { BlockBroker, BlockRetrievalOptions } from '@helia/interface/blocks' import type { Libp2p, Startable } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { Bitswap, BitswapNotifyProgressEvents, BitswapOptions, BitswapWantBlockProgressEvents } from 'ipfs-bitswap' @@ -17,9 +17,7 @@ export interface BitswapInit extends BitswapOptions { } -class BitswapBlockBroker implements BlockAnnouncer>, BlockRetriever< -ProgressOptions ->, Startable { +class BitswapBlockBroker implements BlockBroker, ProgressOptions>, Startable { private readonly bitswap: Bitswap private started: boolean @@ -69,7 +67,7 @@ ProgressOptions this.bitswap.notify(cid, block, options) } - async retrieve (cid: CID, { validateFn, ...options }: BlockRetrievalOptions> = {}): Promise { + async retrieve (cid: CID, options: BlockRetrievalOptions> = {}): Promise { return this.bitswap.want(cid, options) } } diff --git a/packages/block-brokers/src/trustless-gateway/broker.ts b/packages/block-brokers/src/trustless-gateway/broker.ts index 433a690d..a080fde9 100644 --- a/packages/block-brokers/src/trustless-gateway/broker.ts +++ b/packages/block-brokers/src/trustless-gateway/broker.ts @@ -1,7 +1,7 @@ import { TrustlessGateway } from './trustless-gateway.js' import { DEFAULT_TRUSTLESS_GATEWAYS } from './index.js' import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayComponents, TrustlessGatewayGetBlockProgressEvents } from './index.js' -import type { BlockRetrievalOptions, BlockRetriever } from '@helia/interface/blocks' +import type { BlockRetrievalOptions, BlockBroker } from '@helia/interface/blocks' import type { Logger } from '@libp2p/interface' import type { CID } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' @@ -10,7 +10,7 @@ import type { ProgressOptions } from 'progress-events' * A class that accepts a list of trustless gateways that are queried * for blocks. */ -export class TrustlessGatewayBlockBroker implements BlockRetriever< +export class TrustlessGatewayBlockBroker implements BlockBroker< ProgressOptions > { private readonly gateways: TrustlessGateway[] diff --git a/packages/block-brokers/src/trustless-gateway/index.ts b/packages/block-brokers/src/trustless-gateway/index.ts index 91dcab15..5908816c 100644 --- a/packages/block-brokers/src/trustless-gateway/index.ts +++ b/packages/block-brokers/src/trustless-gateway/index.ts @@ -1,7 +1,7 @@ import { TrustlessGatewayBlockBroker } from './broker.js' -import type { BlockRetriever } from '@helia/interface/src/blocks.js' +import type { BlockBroker } from '@helia/interface/src/blocks.js' import type { ComponentLogger } from '@libp2p/interface' -import type { ProgressEvent } from 'progress-events' +import type { ProgressEvent, ProgressOptions } from 'progress-events' export const DEFAULT_TRUSTLESS_GATEWAYS = [ // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/ @@ -25,6 +25,6 @@ export interface TrustlessGatewayComponents { logger: ComponentLogger } -export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockRetriever { +export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockBroker> { return (components) => new TrustlessGatewayBlockBroker(components, init) } diff --git a/packages/block-brokers/test/trustless-gateway.spec.ts b/packages/block-brokers/test/trustless-gateway.spec.ts index 481ac6df..9851e412 100644 --- a/packages/block-brokers/test/trustless-gateway.spec.ts +++ b/packages/block-brokers/test/trustless-gateway.spec.ts @@ -8,12 +8,12 @@ import { type StubbedInstance, stubConstructor } from 'sinon-ts' import { TrustlessGatewayBlockBroker } from '../src/trustless-gateway/broker.js' import { TrustlessGateway } from '../src/trustless-gateway/trustless-gateway.js' import { createBlock } from './fixtures/create-block.js' -import type { BlockRetriever } from '@helia/interface/blocks' +import type { BlockBroker } from '@helia/interface/blocks' import type { CID } from 'multiformats/cid' describe('trustless-gateway-block-broker', () => { let blocks: Array<{ cid: CID, block: Uint8Array }> - let gatewayBlockBroker: BlockRetriever + let gatewayBlockBroker: BlockBroker let gateways: Array> // take a Record) => void> and stub the gateways @@ -54,7 +54,7 @@ describe('trustless-gateway-block-broker', () => { gateway.getRawBlock.rejects(new Error('failed')) } - await expect(gatewayBlockBroker.retrieve(blocks[0].cid)) + await expect(gatewayBlockBroker.retrieve?.(blocks[0].cid)) .to.eventually.be.rejected() .with.property('errors') .with.lengthOf(gateways.length) @@ -78,7 +78,7 @@ describe('trustless-gateway-block-broker', () => { } }) - await expect(gatewayBlockBroker.retrieve(blocks[1].cid)).to.eventually.be.rejected() + await expect(gatewayBlockBroker.retrieve?.(blocks[1].cid)).to.eventually.be.rejected() // all gateways were called expect(gateways[0].getRawBlock.calledWith(blocks[1].cid)).to.be.true() @@ -105,7 +105,7 @@ describe('trustless-gateway-block-broker', () => { } }) - const block = await gatewayBlockBroker.retrieve(cid1, { + const block = await gatewayBlockBroker.retrieve?.(cid1, { validateFn: async (block) => { if (block !== block1) { throw new Error('invalid block') @@ -136,7 +136,7 @@ describe('trustless-gateway-block-broker', () => { gateway.reliability.returns(0) // make sure other gateways are called last } }) - const block = await gatewayBlockBroker.retrieve(cid1, { + const block = await gatewayBlockBroker.retrieve?.(cid1, { validateFn: async (block) => { if (block !== block1) { throw new Error('invalid block') diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index ecd23012..635b0346 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -44,7 +44,9 @@ export type DeleteManyBlocksProgressEvents = export interface GetOfflineOptions { /** - * If true, do not attempt to fetch any missing blocks from the network (default: false) + * If true, do not attempt to fetch any missing blocks from the network + * + * @default false */ offline?: boolean } @@ -54,7 +56,18 @@ ProgressOptions, ProgressOptions, GetOfflineOptions & ProgressOptions, ProgressOptions, ProgressOptions, ProgressOptions > { - + /** + * A session blockstore is a special blockstore that only pulls content from a + * subset of network peers which respond as having the block for the initial + * root CID. + * + * Any blocks written to the blockstore as part of the session will propagate + * to the blockstore the session was created from. + * + * This method is optional to maintain compatibility with existing + * blockstores that do not support sessions. + */ + createSession?(root: CID, options?: AbortOptions & ProgressOptions): Promise } export type BlockRetrievalOptions = AbortOptions & GetProgressOptions & { @@ -67,18 +80,19 @@ export type BlockRetrievalOptions } -export interface BlockRetriever { +export interface BlockBroker { /** * Retrieve a block from a source */ - retrieve(cid: CID, options?: BlockRetrievalOptions): Promise -} + retrieve?(cid: CID, options?: BlockRetrievalOptions): Promise -export interface BlockAnnouncer { /** * Make a new block available to peers */ - announce(cid: CID, block: Uint8Array, options?: NotifyProgressOptions): void -} + announce?(cid: CID, block: Uint8Array, options?: NotifyProgressOptions): void -export type BlockBroker = BlockRetriever | BlockAnnouncer + /** + * Create a new session + */ + createSession?(root: CID, options?: BlockRetrievalOptions): Promise> +} diff --git a/packages/utils/src/storage.ts b/packages/utils/src/storage.ts index 909c2e89..99925108 100644 --- a/packages/utils/src/storage.ts +++ b/packages/utils/src/storage.ts @@ -1,4 +1,4 @@ -import { start, stop } from '@libp2p/interface' +import { CodeError, start, stop } from '@libp2p/interface' import createMortice from 'mortice' import type { Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions } from '@helia/interface/blocks' import type { Pins } from '@helia/interface/pins' @@ -24,14 +24,14 @@ export interface GetOptions extends AbortOptions { */ export class BlockStorage implements Blocks, Startable { public lock: Mortice - private readonly child: Blockstore + private readonly child: Blocks private readonly pins: Pins private started: boolean /** * Create a new BlockStorage */ - constructor (blockstore: Blockstore, pins: Pins, options: BlockStorageInit = {}) { + constructor (blockstore: Blocks, pins: Pins, options: BlockStorageInit = {}) { this.child = blockstore this.pins = pins this.lock = createMortice({ @@ -169,4 +169,20 @@ export class BlockStorage implements Blocks, Startable { releaseLock() } } + + async createSession (root: CID, options?: AbortOptions): Promise { + const releaseLock = await this.lock.readLock() + + try { + const blocks = await this.child.createSession?.(root, options) + + if (blocks == null) { + throw new CodeError('Sessions not supported', 'ERR_UNSUPPORTED') + } + + return blocks + } finally { + releaseLock() + } + } } diff --git a/packages/utils/src/utils/networked-storage.ts b/packages/utils/src/utils/networked-storage.ts index bf98500f..64425620 100644 --- a/packages/utils/src/utils/networked-storage.ts +++ b/packages/utils/src/utils/networked-storage.ts @@ -4,23 +4,19 @@ import filter from 'it-filter' import forEach from 'it-foreach' import { CustomProgressEvent, type ProgressOptions } from 'progress-events' import { equals as uint8ArrayEquals } from 'uint8arrays/equals' -import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetriever, BlockAnnouncer, BlockRetrievalOptions } from '@helia/interface/blocks' +import type { BlockBroker, Blocks, Pair, DeleteManyBlocksProgressEvents, DeleteBlockProgressEvents, GetBlockProgressEvents, GetManyBlocksProgressEvents, PutManyBlocksProgressEvents, PutBlockProgressEvents, GetAllBlocksProgressEvents, GetOfflineOptions, BlockRetrievalOptions } from '@helia/interface/blocks' import type { AbortOptions, ComponentLogger, Logger, LoggerOptions, Startable } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { AwaitIterable } from 'interface-store' import type { CID } from 'multiformats/cid' import type { MultihashHasher } from 'multiformats/hashes/interface' -export interface GetOptions extends AbortOptions { - progress?(evt: Event): void +export interface NetworkedStorageStorageInit { + root?: CID } -function isBlockRetriever (b: any): b is BlockRetriever { - return typeof b.retrieve === 'function' -} - -function isBlockAnnouncer (b: any): b is BlockAnnouncer { - return typeof b.announce === 'function' +export interface GetOptions extends AbortOptions { + progress?(evt: Event): void } export interface NetworkedStorageComponents { @@ -36,20 +32,20 @@ export interface NetworkedStorageComponents { */ export class NetworkedStorage implements Blocks, Startable { private readonly child: Blockstore - private readonly blockRetrievers: BlockRetriever[] - private readonly blockAnnouncers: BlockAnnouncer[] + private readonly blockBrokers: BlockBroker[] private readonly hashers: Record private started: boolean private readonly log: Logger + private readonly logger: ComponentLogger /** * Create a new BlockStorage */ - constructor (components: NetworkedStorageComponents) { - this.log = components.logger.forComponent('helia:networked-storage') + constructor (components: NetworkedStorageComponents, init: NetworkedStorageStorageInit = {}) { + this.log = components.logger.forComponent(`helia:networked-storage${init.root == null ? '' : `:${init.root}`}`) + this.logger = components.logger this.child = components.blockstore - this.blockRetrievers = (components.blockBrokers ?? []).filter(isBlockRetriever) - this.blockAnnouncers = (components.blockBrokers ?? []).filter(isBlockAnnouncer) + this.blockBrokers = components.blockBrokers ?? [] this.hashers = components.hashers ?? {} this.started = false } @@ -59,12 +55,12 @@ export class NetworkedStorage implements Blocks, Startable { } async start (): Promise { - await start(this.child, ...new Set([...this.blockRetrievers, ...this.blockAnnouncers])) + await start(this.child, ...this.blockBrokers) this.started = true } async stop (): Promise { - await stop(this.child, ...new Set([...this.blockRetrievers, ...this.blockAnnouncers])) + await stop(this.child, ...this.blockBrokers) this.started = false } @@ -83,8 +79,8 @@ export class NetworkedStorage implements Blocks, Startable { options.onProgress?.(new CustomProgressEvent('blocks:put:providers:notify', cid)) - this.blockAnnouncers.forEach(provider => { - provider.announce(cid, block, options) + this.blockBrokers.forEach(broker => { + broker.announce?.(cid, block, options) }) options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid)) @@ -108,8 +104,8 @@ export class NetworkedStorage implements Blocks, Startable { const notifyEach = forEach(missingBlocks, ({ cid, block }): void => { options.onProgress?.(new CustomProgressEvent('blocks:put-many:providers:notify', cid)) - this.blockAnnouncers.forEach(provider => { - provider.announce(cid, block, options) + this.blockBrokers.forEach(broker => { + broker.announce?.(cid, block, options) }) }) @@ -124,7 +120,7 @@ export class NetworkedStorage implements Blocks, Startable { if (options.offline !== true && !(await this.child.has(cid))) { // we do not have the block locally, get it from a block provider options.onProgress?.(new CustomProgressEvent('blocks:get:providers:get', cid)) - const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers[cid.multihash.code], { + const block = await raceBlockRetrievers(cid, this.blockBrokers, this.hashers[cid.multihash.code], { ...options, log: this.log }) @@ -133,8 +129,8 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get:providers:notify', cid)) - this.blockAnnouncers.forEach(provider => { - provider.announce(cid, block, options) + this.blockBrokers.forEach(broker => { + broker.announce?.(cid, block, options) }) return block @@ -155,7 +151,7 @@ export class NetworkedStorage implements Blocks, Startable { if (options.offline !== true && !(await this.child.has(cid))) { // we do not have the block locally, get it from a block provider options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:get', cid)) - const block = await raceBlockRetrievers(cid, this.blockRetrievers, this.hashers[cid.multihash.code], { + const block = await raceBlockRetrievers(cid, this.blockBrokers, this.hashers[cid.multihash.code], { ...options, log: this.log }) @@ -164,8 +160,8 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:notify', cid)) - this.blockAnnouncers.forEach(provider => { - provider.announce(cid, block, options) + this.blockBrokers.forEach(broker => { + broker.announce?.(cid, block, options) }) } })) @@ -200,6 +196,25 @@ export class NetworkedStorage implements Blocks, Startable { options.onProgress?.(new CustomProgressEvent('blocks:get-all:blockstore:get-many')) yield * this.child.getAll(options) } + + async createSession (root: CID, options?: AbortOptions & ProgressOptions): Promise { + const blockBrokers = await Promise.all(this.blockBrokers.map(async broker => { + if (broker.createSession == null) { + return broker + } + + return broker.createSession(root, options) + })) + + return new NetworkedStorage({ + blockstore: this.child, + blockBrokers, + hashers: this.hashers, + logger: this.logger + }, { + root + }) + } } export const getCidBlockVerifierFunction = (cid: CID, hasher: MultihashHasher): Required['validateFn'] => { @@ -222,38 +237,49 @@ export const getCidBlockVerifierFunction = (cid: CID, hasher: MultihashHasher): * Race block providers cancelling any pending requests once the block has been * found. */ -async function raceBlockRetrievers (cid: CID, providers: BlockRetriever[], hasher: MultihashHasher, options: AbortOptions & LoggerOptions): Promise { +async function raceBlockRetrievers (cid: CID, blockBrokers: BlockBroker[], hasher: MultihashHasher, options: AbortOptions & LoggerOptions): Promise { const validateFn = getCidBlockVerifierFunction(cid, hasher) const controller = new AbortController() const signal = anySignal([controller.signal, options.signal]) + const retrievers: Array>> = [] + + for (const broker of blockBrokers) { + if (broker.retrieve != null) { + // @ts-expect-error retrieve may be undefined even though we've just + // checked that it isn't + retrievers.push(broker) + } + } + try { return await Promise.any( - providers.map(async provider => { - try { - let blocksWereValidated = false - const block = await provider.retrieve(cid, { - ...options, - signal, - validateFn: async (block: Uint8Array): Promise => { + retrievers + .map(async retriever => { + try { + let blocksWereValidated = false + const block = await retriever.retrieve(cid, { + ...options, + signal, + validateFn: async (block: Uint8Array): Promise => { + await validateFn(block) + blocksWereValidated = true + } + }) + + if (!blocksWereValidated) { + // the blockBroker either did not throw an error when attempting to validate the block + // or did not call the validateFn at all. We should validate the block ourselves await validateFn(block) - blocksWereValidated = true } - }) - if (!blocksWereValidated) { - // the blockBroker either did not throw an error when attempting to validate the block - // or did not call the validateFn at all. We should validate the block ourselves - await validateFn(block) + return block + } catch (err) { + options.log.error('could not retrieve verified block for %c', cid, err) + throw err } - - return block - } catch (err) { - options.log.error('could not retrieve verified block for %c', cid, err) - throw err - } - }) + }) ) } finally { signal.clear() diff --git a/packages/utils/test/block-broker.spec.ts b/packages/utils/test/block-broker.spec.ts index 1cef6c47..140b7a51 100644 --- a/packages/utils/test/block-broker.spec.ts +++ b/packages/utils/test/block-broker.spec.ts @@ -12,7 +12,7 @@ import { type StubbedInstance, stubInterface } from 'sinon-ts' import { defaultHashers } from '../src/utils/default-hashers.js' import { NetworkedStorage } from '../src/utils/networked-storage.js' import { createBlock } from './fixtures/create-block.js' -import type { BlockBroker, BlockRetriever } from '@helia/interface/blocks' +import type { BlockBroker } from '@helia/interface/blocks' import type { Blockstore } from 'interface-blockstore' import type { CID } from 'multiformats/cid' @@ -21,7 +21,7 @@ describe('block-broker', () => { let blockstore: Blockstore let bitswapBlockBroker: StubbedInstance> let blocks: Array<{ cid: CID, block: Uint8Array }> - let gatewayBlockBroker: StubbedInstance> + let gatewayBlockBroker: StubbedInstance> beforeEach(async () => { blocks = [] diff --git a/packages/utils/test/storage.spec.ts b/packages/utils/test/storage.spec.ts index 92d7614a..db93c844 100644 --- a/packages/utils/test/storage.spec.ts +++ b/packages/utils/test/storage.spec.ts @@ -10,13 +10,13 @@ import * as raw from 'multiformats/codecs/raw' import { PinsImpl } from '../src/pins.js' import { BlockStorage } from '../src/storage.js' import { createBlock } from './fixtures/create-block.js' +import type { Blocks } from '@helia/interface' import type { Pins } from '@helia/interface/pins' -import type { Blockstore } from 'interface-blockstore' import type { CID } from 'multiformats/cid' describe('storage', () => { let storage: BlockStorage - let blockstore: Blockstore + let blockstore: Blocks let pins: Pins let blocks: Array<{ cid: CID, block: Uint8Array }> diff --git a/packages/utils/test/utils/networked-storage.spec.ts b/packages/utils/test/utils/networked-storage.spec.ts index 46a7d4ad..46c98fe6 100644 --- a/packages/utils/test/utils/networked-storage.spec.ts +++ b/packages/utils/test/utils/networked-storage.spec.ts @@ -12,14 +12,14 @@ import { type StubbedInstance, stubInterface } from 'sinon-ts' import { defaultHashers } from '../../src/utils/default-hashers.js' import { NetworkedStorage } from '../../src/utils/networked-storage.js' import { createBlock } from '../fixtures/create-block.js' -import type { BlockAnnouncer, BlockRetriever } from '@helia/interface/blocks' +import type { BlockBroker } from '@helia/interface/blocks' import type { Blockstore } from 'interface-blockstore' import type { CID } from 'multiformats/cid' describe('networked-storage', () => { let storage: NetworkedStorage let blockstore: Blockstore - let bitswap: StubbedInstance> + let bitswap: StubbedInstance> let blocks: Array<{ cid: CID, block: Uint8Array }> beforeEach(async () => { From 5836dcdec0c26fd62a64d8c4776df2dbd53a33c9 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 30 Jan 2024 16:40:37 +0100 Subject: [PATCH 02/27] chore: pr comments --- packages/block-brokers/src/bitswap.ts | 2 +- packages/interface/src/blocks.ts | 2 +- packages/utils/src/utils/networked-storage.ts | 34 +++++++++++-------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/packages/block-brokers/src/bitswap.ts b/packages/block-brokers/src/bitswap.ts index f764e233..aae1c7b7 100644 --- a/packages/block-brokers/src/bitswap.ts +++ b/packages/block-brokers/src/bitswap.ts @@ -63,7 +63,7 @@ class BitswapBlockBroker implements BlockBroker): void { + async announce (cid: CID, block: Uint8Array, options?: ProgressOptions): Promise { this.bitswap.notify(cid, block, options) } diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index 635b0346..f55f6df7 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -89,7 +89,7 @@ export interface BlockBroker /** * Create a new session diff --git a/packages/utils/src/utils/networked-storage.ts b/packages/utils/src/utils/networked-storage.ts index 64425620..692f7726 100644 --- a/packages/utils/src/utils/networked-storage.ts +++ b/packages/utils/src/utils/networked-storage.ts @@ -11,7 +11,7 @@ import type { AwaitIterable } from 'interface-store' import type { CID } from 'multiformats/cid' import type { MultihashHasher } from 'multiformats/hashes/interface' -export interface NetworkedStorageStorageInit { +export interface NetworkedStorageInit { root?: CID } @@ -41,7 +41,7 @@ export class NetworkedStorage implements Blocks, Startable { /** * Create a new BlockStorage */ - constructor (components: NetworkedStorageComponents, init: NetworkedStorageStorageInit = {}) { + constructor (components: NetworkedStorageComponents, init: NetworkedStorageInit = {}) { this.log = components.logger.forComponent(`helia:networked-storage${init.root == null ? '' : `:${init.root}`}`) this.logger = components.logger this.child = components.blockstore @@ -79,9 +79,13 @@ export class NetworkedStorage implements Blocks, Startable { options.onProgress?.(new CustomProgressEvent('blocks:put:providers:notify', cid)) - this.blockBrokers.forEach(broker => { - broker.announce?.(cid, block, options) - }) + await Promise.all( + this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + ) + + await Promise.all( + this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + ) options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid)) @@ -102,11 +106,11 @@ export class NetworkedStorage implements Blocks, Startable { return !has }) - const notifyEach = forEach(missingBlocks, ({ cid, block }): void => { + const notifyEach = forEach(missingBlocks, async ({ cid, block }): Promise => { options.onProgress?.(new CustomProgressEvent('blocks:put-many:providers:notify', cid)) - this.blockBrokers.forEach(broker => { - broker.announce?.(cid, block, options) - }) + await Promise.all( + this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + ) }) options.onProgress?.(new CustomProgressEvent('blocks:put-many:blockstore:put-many')) @@ -129,9 +133,9 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get:providers:notify', cid)) - this.blockBrokers.forEach(broker => { - broker.announce?.(cid, block, options) - }) + await Promise.all( + this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + ) return block } @@ -160,9 +164,9 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:notify', cid)) - this.blockBrokers.forEach(broker => { - broker.announce?.(cid, block, options) - }) + await Promise.all( + this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + ) } })) } From 3d9cb49f4153acd74cc48225fe9c699295a210f3 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 30 Jan 2024 17:08:10 +0100 Subject: [PATCH 03/27] chore: linting --- packages/utils/src/utils/networked-storage.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/utils/src/utils/networked-storage.ts b/packages/utils/src/utils/networked-storage.ts index 692f7726..3f012174 100644 --- a/packages/utils/src/utils/networked-storage.ts +++ b/packages/utils/src/utils/networked-storage.ts @@ -80,11 +80,11 @@ export class NetworkedStorage implements Blocks, Startable { options.onProgress?.(new CustomProgressEvent('blocks:put:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) await Promise.all( - this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid)) @@ -109,7 +109,7 @@ export class NetworkedStorage implements Blocks, Startable { const notifyEach = forEach(missingBlocks, async ({ cid, block }): Promise => { options.onProgress?.(new CustomProgressEvent('blocks:put-many:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) }) @@ -134,7 +134,7 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) return block @@ -165,7 +165,7 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(broker => broker.announce?.(cid, block, options)) + this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) } })) From 39b6f7a68ba55b1b3571e61d89277fed07f58e73 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 5 Feb 2024 18:11:18 +0100 Subject: [PATCH 04/27] chore: simplify options --- packages/block-brokers/src/bitswap.ts | 9 +++--- .../src/trustless-gateway/broker.ts | 7 ++-- .../src/trustless-gateway/index.ts | 4 +-- packages/interface/src/blocks.ts | 32 +++++++++++++++---- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/packages/block-brokers/src/bitswap.ts b/packages/block-brokers/src/bitswap.ts index aae1c7b7..dfe79ff1 100644 --- a/packages/block-brokers/src/bitswap.ts +++ b/packages/block-brokers/src/bitswap.ts @@ -1,11 +1,10 @@ import { createBitswap } from 'ipfs-bitswap' -import type { BlockBroker, BlockRetrievalOptions } from '@helia/interface/blocks' +import type { BlockAnnounceOptions, BlockBroker, BlockRetrievalOptions } from '@helia/interface/blocks' import type { Libp2p, Startable } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { Bitswap, BitswapNotifyProgressEvents, BitswapOptions, BitswapWantBlockProgressEvents } from 'ipfs-bitswap' import type { CID } from 'multiformats/cid' import type { MultihashHasher } from 'multiformats/hashes/interface' -import type { ProgressOptions } from 'progress-events' interface BitswapComponents { libp2p: Libp2p @@ -17,7 +16,7 @@ export interface BitswapInit extends BitswapOptions { } -class BitswapBlockBroker implements BlockBroker, ProgressOptions>, Startable { +class BitswapBlockBroker implements BlockBroker, Startable { private readonly bitswap: Bitswap private started: boolean @@ -63,11 +62,11 @@ class BitswapBlockBroker implements BlockBroker): Promise { + async announce (cid: CID, block: Uint8Array, options?: BlockAnnounceOptions): Promise { this.bitswap.notify(cid, block, options) } - async retrieve (cid: CID, options: BlockRetrievalOptions> = {}): Promise { + async retrieve (cid: CID, options: BlockRetrievalOptions = {}): Promise { return this.bitswap.want(cid, options) } } diff --git a/packages/block-brokers/src/trustless-gateway/broker.ts b/packages/block-brokers/src/trustless-gateway/broker.ts index a080fde9..d1954816 100644 --- a/packages/block-brokers/src/trustless-gateway/broker.ts +++ b/packages/block-brokers/src/trustless-gateway/broker.ts @@ -4,15 +4,12 @@ import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayComponents, Trust import type { BlockRetrievalOptions, BlockBroker } from '@helia/interface/blocks' import type { Logger } from '@libp2p/interface' import type { CID } from 'multiformats/cid' -import type { ProgressOptions } from 'progress-events' /** * A class that accepts a list of trustless gateways that are queried * for blocks. */ -export class TrustlessGatewayBlockBroker implements BlockBroker< -ProgressOptions -> { +export class TrustlessGatewayBlockBroker implements BlockBroker { private readonly gateways: TrustlessGateway[] private readonly log: Logger @@ -24,7 +21,7 @@ ProgressOptions }) } - async retrieve (cid: CID, options: BlockRetrievalOptions> = {}): Promise { + async retrieve (cid: CID, options: BlockRetrievalOptions = {}): Promise { // Loop through the gateways until we get a block or run out of gateways // TODO: switch to toSorted when support is better const sortedGateways = this.gateways.sort((a, b) => b.reliability() - a.reliability()) diff --git a/packages/block-brokers/src/trustless-gateway/index.ts b/packages/block-brokers/src/trustless-gateway/index.ts index 5908816c..93489ecb 100644 --- a/packages/block-brokers/src/trustless-gateway/index.ts +++ b/packages/block-brokers/src/trustless-gateway/index.ts @@ -1,7 +1,7 @@ import { TrustlessGatewayBlockBroker } from './broker.js' import type { BlockBroker } from '@helia/interface/src/blocks.js' import type { ComponentLogger } from '@libp2p/interface' -import type { ProgressEvent, ProgressOptions } from 'progress-events' +import type { ProgressEvent } from 'progress-events' export const DEFAULT_TRUSTLESS_GATEWAYS = [ // 2023-10-03: IPNS, Origin, and Block/CAR support from https://ipfs-public-gateway-checker.on.fleek.co/ @@ -25,6 +25,6 @@ export interface TrustlessGatewayComponents { logger: ComponentLogger } -export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockBroker> { +export function trustlessGateway (init: TrustlessGatewayBlockBrokerInit = {}): (components: TrustlessGatewayComponents) => BlockBroker { return (components) => new TrustlessGatewayBlockBroker(components, init) } diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index f55f6df7..377fc804 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -67,10 +67,10 @@ ProgressOptions, ProgressOptions): Promise + createSession?(root: CID, options?: CreateSessionOptions): Promise } -export type BlockRetrievalOptions = AbortOptions & GetProgressOptions & { +export interface BlockRetrievalOptions = ProgressEvent> extends AbortOptions, ProgressOptions { /** * A function that blockBrokers should call prior to returning a block to ensure it can maintain control * of the block request flow. e.g. TrustedGatewayBlockBroker will use this to ensure that the block @@ -80,19 +80,39 @@ export type BlockRetrievalOptions } -export interface BlockBroker { +export interface BlockAnnounceOptions = ProgressEvent> extends AbortOptions, ProgressOptions { + +} + +export interface CreateSessionOptions = ProgressEvent> extends AbortOptions, ProgressOptions { + /** + * The minimum number of providers for the root CID that are required for + * successful session creation. + */ + providers?: number + + /** + * How long each queried provider has to respond either that they have the + * root block or to send it to us. + * + * @default 5000 + */ + timeout?: number +} + +export interface BlockBroker = ProgressEvent, AnnounceProgressEvents extends ProgressEvent = ProgressEvent> { /** * Retrieve a block from a source */ - retrieve?(cid: CID, options?: BlockRetrievalOptions): Promise + retrieve?(cid: CID, options?: BlockRetrievalOptions): Promise /** * Make a new block available to peers */ - announce?(cid: CID, block: Uint8Array, options?: NotifyProgressOptions): Promise + announce?(cid: CID, block: Uint8Array, options?: BlockAnnounceOptions): Promise /** * Create a new session */ - createSession?(root: CID, options?: BlockRetrievalOptions): Promise> + createSession?(root: CID, options?: CreateSessionOptions): Promise> } From d1228b9d69818141c6ec38f0131449ff0a1301ce Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 5 Feb 2024 18:38:00 +0100 Subject: [PATCH 05/27] chore: createSession is not optional on blocks --- packages/interface/src/blocks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index 377fc804..fd283dd6 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -67,7 +67,7 @@ ProgressOptions, ProgressOptions): Promise + createSession(root: CID, options?: CreateSessionOptions): Promise } export interface BlockRetrievalOptions = ProgressEvent> extends AbortOptions, ProgressOptions { From 32713ab2e0ce25187a601f4a5907ef74f1a1c79d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 6 Feb 2024 08:59:03 +0100 Subject: [PATCH 06/27] chore: update utils --- packages/utils/src/storage.ts | 4 ++-- packages/utils/src/utils/networked-storage.ts | 4 ---- packages/utils/test/storage.spec.ts | 9 ++++++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/utils/src/storage.ts b/packages/utils/src/storage.ts index 99925108..f184ae22 100644 --- a/packages/utils/src/storage.ts +++ b/packages/utils/src/storage.ts @@ -170,11 +170,11 @@ export class BlockStorage implements Blocks, Startable { } } - async createSession (root: CID, options?: AbortOptions): Promise { + async createSession (root: CID, options?: AbortOptions): Promise { const releaseLock = await this.lock.readLock() try { - const blocks = await this.child.createSession?.(root, options) + const blocks = await this.child.createSession(root, options) if (blocks == null) { throw new CodeError('Sessions not supported', 'ERR_UNSUPPORTED') diff --git a/packages/utils/src/utils/networked-storage.ts b/packages/utils/src/utils/networked-storage.ts index 3f012174..57b58c78 100644 --- a/packages/utils/src/utils/networked-storage.ts +++ b/packages/utils/src/utils/networked-storage.ts @@ -83,10 +83,6 @@ export class NetworkedStorage implements Blocks, Startable { this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) - await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) - ) - options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid)) return this.child.put(cid, block, options) diff --git a/packages/utils/test/storage.spec.ts b/packages/utils/test/storage.spec.ts index db93c844..42c5a5b4 100644 --- a/packages/utils/test/storage.spec.ts +++ b/packages/utils/test/storage.spec.ts @@ -12,8 +12,15 @@ import { BlockStorage } from '../src/storage.js' import { createBlock } from './fixtures/create-block.js' import type { Blocks } from '@helia/interface' import type { Pins } from '@helia/interface/pins' +import type { Blockstore } from 'interface-blockstore' import type { CID } from 'multiformats/cid' +class MemoryBlocks extends MemoryBlockstore implements Blocks { + async createSession (): Promise { + throw new Error('Not implemented') + } +} + describe('storage', () => { let storage: BlockStorage let blockstore: Blocks @@ -29,7 +36,7 @@ describe('storage', () => { const datastore = new MemoryDatastore() - blockstore = new MemoryBlockstore() + blockstore = new MemoryBlocks() pins = new PinsImpl(datastore, blockstore, []) storage = new BlockStorage(blockstore, pins, { holdGcLock: true From e4857225ddcfd2602c084f2a5b5417d44917bdd4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 30 Jan 2024 15:25:28 +0100 Subject: [PATCH 07/27] feat: add @helia/bitswap with sessions Adds a `@helia/bitswap` module with code ported from `ipfs-bitswap` and greatly simplified. - Supports sessions - Only supports bitswap 1.2.0 - Uses libp2p's metrics system instead of a custom version --- .release-please-manifest.json | 1 + .release-please.json | 1 + packages/bitswap/.aegir.js | 7 + packages/bitswap/CHANGELOG.md | 902 ++++++++++++++++++ packages/bitswap/LICENSE | 4 + packages/bitswap/LICENSE-APACHE | 5 + packages/bitswap/LICENSE-MIT | 19 + packages/bitswap/README.md | 47 + packages/bitswap/package.json | 204 ++++ packages/bitswap/src/bitswap.ts | 481 ++++++++++ packages/bitswap/src/constants.ts | 12 + packages/bitswap/src/index.ts | 236 +++++ packages/bitswap/src/network.ts | 476 +++++++++ packages/bitswap/src/notifications.ts | 137 +++ packages/bitswap/src/pb/message.proto | 42 + packages/bitswap/src/pb/message.ts | 450 +++++++++ packages/bitswap/src/peer-want-lists/index.ts | 148 +++ .../bitswap/src/peer-want-lists/ledger.ts | 138 +++ packages/bitswap/src/session.ts | 67 ++ packages/bitswap/src/stats.ts | 70 ++ packages/bitswap/src/utils/cid-prefix.ts | 8 + packages/bitswap/src/utils/varint-decoder.ts | 19 + packages/bitswap/src/utils/varint-encoder.ts | 18 + packages/bitswap/src/want-list.ts | 271 ++++++ packages/bitswap/test/bitswap.spec.ts | 451 +++++++++ packages/bitswap/test/network.spec.ts | 445 +++++++++ packages/bitswap/test/notifications.spec.ts | 71 ++ packages/bitswap/test/peer-want-list.spec.ts | 501 ++++++++++ packages/bitswap/test/session.spec.ts | 60 ++ packages/bitswap/test/stats.spec.ts | 104 ++ packages/bitswap/test/want-list.spec.ts | 126 +++ packages/bitswap/tsconfig.json | 15 + packages/bitswap/typedoc.json | 5 + packages/block-brokers/package.json | 1 + packages/block-brokers/src/bitswap.ts | 28 +- packages/block-brokers/tsconfig.json | 3 + 36 files changed, 5567 insertions(+), 6 deletions(-) create mode 100644 packages/bitswap/.aegir.js create mode 100644 packages/bitswap/CHANGELOG.md create mode 100644 packages/bitswap/LICENSE create mode 100644 packages/bitswap/LICENSE-APACHE create mode 100644 packages/bitswap/LICENSE-MIT create mode 100644 packages/bitswap/README.md create mode 100644 packages/bitswap/package.json create mode 100644 packages/bitswap/src/bitswap.ts create mode 100644 packages/bitswap/src/constants.ts create mode 100644 packages/bitswap/src/index.ts create mode 100644 packages/bitswap/src/network.ts create mode 100644 packages/bitswap/src/notifications.ts create mode 100644 packages/bitswap/src/pb/message.proto create mode 100644 packages/bitswap/src/pb/message.ts create mode 100644 packages/bitswap/src/peer-want-lists/index.ts create mode 100644 packages/bitswap/src/peer-want-lists/ledger.ts create mode 100644 packages/bitswap/src/session.ts create mode 100644 packages/bitswap/src/stats.ts create mode 100644 packages/bitswap/src/utils/cid-prefix.ts create mode 100644 packages/bitswap/src/utils/varint-decoder.ts create mode 100644 packages/bitswap/src/utils/varint-encoder.ts create mode 100644 packages/bitswap/src/want-list.ts create mode 100644 packages/bitswap/test/bitswap.spec.ts create mode 100644 packages/bitswap/test/network.spec.ts create mode 100644 packages/bitswap/test/notifications.spec.ts create mode 100644 packages/bitswap/test/peer-want-list.spec.ts create mode 100644 packages/bitswap/test/session.spec.ts create mode 100644 packages/bitswap/test/stats.spec.ts create mode 100644 packages/bitswap/test/want-list.spec.ts create mode 100644 packages/bitswap/tsconfig.json create mode 100644 packages/bitswap/typedoc.json diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 3bab71b9..971c6449 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,5 +1,6 @@ { "packages/block-brokers": "2.0.1", + "packages/bitswap": "0.0.0", "packages/car": "3.0.0", "packages/dag-cbor": "3.0.0", "packages/dag-json": "3.0.0", diff --git a/.release-please.json b/.release-please.json index f38bc1fb..2fd890c4 100644 --- a/.release-please.json +++ b/.release-please.json @@ -10,6 +10,7 @@ ], "packages": { "packages/block-brokers": {}, + "packages/bitswap": {}, "packages/car": {}, "packages/dag-cbor": {}, "packages/dag-json": {}, diff --git a/packages/bitswap/.aegir.js b/packages/bitswap/.aegir.js new file mode 100644 index 00000000..b7d54c2f --- /dev/null +++ b/packages/bitswap/.aegir.js @@ -0,0 +1,7 @@ + +/** @type {import('aegir').PartialOptions} */ +export default { + build: { + bundlesizeMax: '33KB' + } +} diff --git a/packages/bitswap/CHANGELOG.md b/packages/bitswap/CHANGELOG.md new file mode 100644 index 00000000..f9d785c9 --- /dev/null +++ b/packages/bitswap/CHANGELOG.md @@ -0,0 +1,902 @@ +## [20.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v19.0.2...v20.0.0) (2023-11-30) + + +### ⚠ BREAKING CHANGES + +* requires libp2p v1 + +### Dependencies + +* update libp2p to v1 ([#610](https://github.com/ipfs/js-ipfs-bitswap/issues/610)) ([9f8258d](https://github.com/ipfs/js-ipfs-bitswap/commit/9f8258da440f6b2c5064687bf10136c785344234)) + +## [19.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v19.0.1...v19.0.2) (2023-11-04) + + +### Dependencies + +* **dev:** bump sinon from 16.1.3 to 17.0.1 ([#606](https://github.com/ipfs/js-ipfs-bitswap/issues/606)) ([01f8738](https://github.com/ipfs/js-ipfs-bitswap/commit/01f8738df272bed69bcb2714dbe80c3e28f4d7b6)) + +## [19.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v19.0.0...v19.0.1) (2023-10-09) + + +### Trivial Changes + +* add or force update .github/workflows/js-test-and-release.yml ([#598](https://github.com/ipfs/js-ipfs-bitswap/issues/598)) ([061acfa](https://github.com/ipfs/js-ipfs-bitswap/commit/061acfa2ffb8e1893bb3a26ebd320be20973e322)) +* delete templates [skip ci] ([#597](https://github.com/ipfs/js-ipfs-bitswap/issues/597)) ([0d56b0a](https://github.com/ipfs/js-ipfs-bitswap/commit/0d56b0aaa6de9a58f2b73e72784ad49ba4b2db45)) + + +### Dependencies + +* **dev:** bump aegir from 40.0.13 to 41.0.0 ([#601](https://github.com/ipfs/js-ipfs-bitswap/issues/601)) ([a510fdc](https://github.com/ipfs/js-ipfs-bitswap/commit/a510fdc7b149b94bc3043c4f5b25700270a5408f)) +* **dev:** bump sinon from 15.2.0 to 16.1.0 ([#602](https://github.com/ipfs/js-ipfs-bitswap/issues/602)) ([ac858e7](https://github.com/ipfs/js-ipfs-bitswap/commit/ac858e7d8edcf965da150d89adc0f7a470f2899a)) + +## [19.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v18.0.3...v19.0.0) (2023-08-05) + + +### ⚠ BREAKING CHANGES + +* requires libp2p@0.46.x or later + +### Dependencies + +* update to libp2p 0.46.x ([#596](https://github.com/ipfs/js-ipfs-bitswap/issues/596)) ([be8fe06](https://github.com/ipfs/js-ipfs-bitswap/commit/be8fe06bb10d6d940ac51c56b30c80154469673b)) + +## [18.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v18.0.2...v18.0.3) (2023-07-27) + + +### Dependencies + +* **dev:** bump aegir from 39.0.13 to 40.0.1 ([#585](https://github.com/ipfs/js-ipfs-bitswap/issues/585)) ([09755bd](https://github.com/ipfs/js-ipfs-bitswap/commit/09755bd46a4fa7599ffdabc35d3ac7d8115abc07)) + +## [18.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v18.0.1...v18.0.2) (2023-07-27) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([79974a5](https://github.com/ipfs/js-ipfs-bitswap/commit/79974a5b165d84b878f25bb11ac63639d4e3e093)) +* Update .github/workflows/stale.yml [skip ci] ([63f3993](https://github.com/ipfs/js-ipfs-bitswap/commit/63f39935cfd560319b6eaf117e539bd018316dc7)) + + +### Dependencies + +* **dev:** bump delay from 5.0.0 to 6.0.0 ([#580](https://github.com/ipfs/js-ipfs-bitswap/issues/580)) ([d796ebc](https://github.com/ipfs/js-ipfs-bitswap/commit/d796ebcd7eb0267cc2709017ec06faf16a4ca2bb)) +* **dev:** bump p-event from 5.0.1 to 6.0.0 ([#582](https://github.com/ipfs/js-ipfs-bitswap/issues/582)) ([ae8fd6f](https://github.com/ipfs/js-ipfs-bitswap/commit/ae8fd6f16bdc49779b78fba3a22975153c138f7e)) + +## [18.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v18.0.0...v18.0.1) (2023-05-22) + + +### Bug Fixes + +* add events dep ([#581](https://github.com/ipfs/js-ipfs-bitswap/issues/581)) ([d26cd16](https://github.com/ipfs/js-ipfs-bitswap/commit/d26cd1642e6abc6c0055451a99ac2daf5b1ad341)) + +## [18.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v17.0.3...v18.0.0) (2023-05-19) + + +### ⚠ BREAKING CHANGES + +* bump libp2p from 0.43.4 to 0.45.1 (#579) + +### Dependencies + +* bump libp2p from 0.43.4 to 0.45.1 ([#579](https://github.com/ipfs/js-ipfs-bitswap/issues/579)) ([90691b9](https://github.com/ipfs/js-ipfs-bitswap/commit/90691b911ea3fedbc030fc2582939d8b6a7e80fc)) + +## [17.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v17.0.2...v17.0.3) (2023-05-19) + + +### Dependencies + +* **dev:** bump aegir from 38.1.8 to 39.0.7 ([#576](https://github.com/ipfs/js-ipfs-bitswap/issues/576)) ([1868cac](https://github.com/ipfs/js-ipfs-bitswap/commit/1868cac8c0934286b91ad05e4b1a38481a3b49b8)) + +## [17.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v17.0.1...v17.0.2) (2023-04-13) + + +### Bug Fixes + +* increase default stream limit ([#550](https://github.com/ipfs/js-ipfs-bitswap/issues/550)) ([3484be0](https://github.com/ipfs/js-ipfs-bitswap/commit/3484be038321e9c2828bded225f4c6dc94a4d5d9)) + +## [17.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v17.0.0...v17.0.1) (2023-04-04) + + +### Bug Fixes + +* add missing events dep ([#548](https://github.com/ipfs/js-ipfs-bitswap/issues/548)) ([eaf862a](https://github.com/ipfs/js-ipfs-bitswap/commit/eaf862a54b6f498431e1bac33c8e0b94e9b43e01)) + +## [17.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v16.0.0...v17.0.0) (2023-03-13) + + +### ⚠ BREAKING CHANGES + +* `.get`, `.getMany`, `.put` and `.putMany` are no longer part of the `Bitswap` interface - instead call `.want` and `.notify` + +### Features + +* simplify bitswap interface, add progress handlers ([#527](https://github.com/ipfs/js-ipfs-bitswap/issues/527)) ([1f31995](https://github.com/ipfs/js-ipfs-bitswap/commit/1f3199505ac53e3c16cb8ea713d2279fbe69acb1)) + +## [16.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v15.0.2...v16.0.0) (2023-02-13) + + +### ⚠ BREAKING CHANGES + +* this module is now typescript + +### Features + +* convert to typescript ([#525](https://github.com/ipfs/js-ipfs-bitswap/issues/525)) ([11d9261](https://github.com/ipfs/js-ipfs-bitswap/commit/11d9261bf26c722a5fa28fc7c9d52c7090cb12a6)) + + +### Trivial Changes + +* update paths ([3874ec4](https://github.com/ipfs/js-ipfs-bitswap/commit/3874ec451fb714ae7e48a4a0b9f69a8187a82255)) + +## [15.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v15.0.1...v15.0.2) (2023-01-27) + + +### Bug Fixes + +* implement .has method from the blockstore interface ([#520](https://github.com/ipfs/js-ipfs-bitswap/issues/520)) ([6cd37ac](https://github.com/ipfs/js-ipfs-bitswap/commit/6cd37ac05a3e5a3e0c1f3e11c9ba73afea7c29d9)) + + +### Trivial Changes + +* remove rimraf as it is not used ([#521](https://github.com/ipfs/js-ipfs-bitswap/issues/521)) ([eac64fd](https://github.com/ipfs/js-ipfs-bitswap/commit/eac64fd1a202ddcbacf73e0e1d3f52c1286b0503)) + +## [15.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v15.0.0...v15.0.1) (2023-01-27) + + +### Dependencies + +* **dev:** bump @chainsafe/libp2p-noise from 10.2.0 to 11.0.0 ([#511](https://github.com/ipfs/js-ipfs-bitswap/issues/511)) ([584db6c](https://github.com/ipfs/js-ipfs-bitswap/commit/584db6c1e226e6f9a5415236612b03ec38523d14)) +* **dev:** bump aegir from 37.12.1 to 38.1.0 ([#513](https://github.com/ipfs/js-ipfs-bitswap/issues/513)) ([72d6a4c](https://github.com/ipfs/js-ipfs-bitswap/commit/72d6a4c48f49583a41967200e567656cef70a9fe)) + +## [15.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v14.0.0...v15.0.0) (2023-01-07) + + +### ⚠ BREAKING CHANGES + +* update multiformats to v11 (#509) + +### Dependencies + +* update multiformats to v11 ([#509](https://github.com/ipfs/js-ipfs-bitswap/issues/509)) ([09d4ff9](https://github.com/ipfs/js-ipfs-bitswap/commit/09d4ff948a9292df03c6205d2e9b3545e166509c)) + +## [14.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v13.0.0...v14.0.0) (2022-11-19) + + +### ⚠ BREAKING CHANGES + +* updates to the new metrics interface + +### Bug Fixes + +* Update to new metrics interface ([#502](https://github.com/ipfs/js-ipfs-bitswap/issues/502)) ([60a5e6c](https://github.com/ipfs/js-ipfs-bitswap/commit/60a5e6cdb3fdb06ef9c8b935b339d2d3e5f8ef07)) + +## [13.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.6...v13.0.0) (2022-10-18) + + +### ⚠ BREAKING CHANGES + +* updates to incompatible multiformats version + +### Dependencies + +* update multiformats to 10.x.x ([#490](https://github.com/ipfs/js-ipfs-bitswap/issues/490)) ([123f06b](https://github.com/ipfs/js-ipfs-bitswap/commit/123f06b372800e5a8a993cd959dcf216f296f320)) + +## [12.0.6](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.5...v12.0.6) (2022-09-21) + + +### Dependencies + +* update @multiformats/multiaddr to 11.0.0 ([#478](https://github.com/ipfs/js-ipfs-bitswap/issues/478)) ([259b69c](https://github.com/ipfs/js-ipfs-bitswap/commit/259b69cb863d63aa61a254a945805ec9a9bef78c)) + +## [12.0.5](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.4...v12.0.5) (2022-09-01) + + +### Bug Fixes + +* reset timeout controller when messages are received ([#474](https://github.com/ipfs/js-ipfs-bitswap/issues/474)) ([f6c6317](https://github.com/ipfs/js-ipfs-bitswap/commit/f6c6317c878a75a760e8d46d4b53e2530631d42b)) + + +### Trivial Changes + +* update project ([#473](https://github.com/ipfs/js-ipfs-bitswap/issues/473)) ([40376cf](https://github.com/ipfs/js-ipfs-bitswap/commit/40376cffd144583dd7f72644a32c79cd4ce5acd0)) + +## [12.0.4](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.3...v12.0.4) (2022-08-17) + + +### Bug Fixes + +* ensure stream is closed when protocol is incorrect ([#471](https://github.com/ipfs/js-ipfs-bitswap/issues/471)) ([2509772](https://github.com/ipfs/js-ipfs-bitswap/commit/2509772baa31bea6ad610a721a4eb31dfa6f125f)) + +## [12.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.2...v12.0.3) (2022-08-15) + + +### Bug Fixes + +* close streams after use ([#470](https://github.com/ipfs/js-ipfs-bitswap/issues/470)) ([a10e9e9](https://github.com/ipfs/js-ipfs-bitswap/commit/a10e9e921d50a4f50ab4e665e1be35f3e4767b56)) + + +### Dependencies + +* bump blockstore-core from 1.0.5 to 2.0.1 ([#469](https://github.com/ipfs/js-ipfs-bitswap/issues/469)) ([2c10911](https://github.com/ipfs/js-ipfs-bitswap/commit/2c109113da63ffb339501740a31b768d8053ea1e)) +* bump interface-blockstore from 2.0.3 to 3.0.0 ([#467](https://github.com/ipfs/js-ipfs-bitswap/issues/467)) ([2f238a9](https://github.com/ipfs/js-ipfs-bitswap/commit/2f238a9ca20eb5402a8f1e3b800668cf0d46d613)) +* **dev:** bump interface-datastore from 6.1.1 to 7.0.0 ([#468](https://github.com/ipfs/js-ipfs-bitswap/issues/468)) ([e7852d1](https://github.com/ipfs/js-ipfs-bitswap/commit/e7852d117b57acb81161a9e76925810407028d59)) + +## [12.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.1...v12.0.2) (2022-08-11) + + +### Bug Fixes + +* time out slow senders ([#455](https://github.com/ipfs/js-ipfs-bitswap/issues/455)) ([1a14c92](https://github.com/ipfs/js-ipfs-bitswap/commit/1a14c92347a1e18754b51b69bd5bbfe5b595e88d)) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([b590c92](https://github.com/ipfs/js-ipfs-bitswap/commit/b590c92f7102326cef251f07c1e6b9f5758d60d8)) +* update project config ([#466](https://github.com/ipfs/js-ipfs-bitswap/issues/466)) ([799e6b0](https://github.com/ipfs/js-ipfs-bitswap/commit/799e6b03df4785e2bf1664771a18d8fcf42fa654)) + + +### Dependencies + +* update protobufs, it-pipe, etc ([#465](https://github.com/ipfs/js-ipfs-bitswap/issues/465)) ([019d5e7](https://github.com/ipfs/js-ipfs-bitswap/commit/019d5e72ecbb2c91f48c3a3d808185d8b7e754d3)) + +## [12.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.0...v12.0.1) (2022-06-29) + + +### Trivial Changes + +* update deps ([#454](https://github.com/ipfs/js-ipfs-bitswap/issues/454)) ([7516593](https://github.com/ipfs/js-ipfs-bitswap/commit/751659351640756d82b9c7ee21bea57efacea877)) + +## [12.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.4...v12.0.0) (2022-06-28) + + +### ⚠ BREAKING CHANGES + +* uses libp2p with protocol stream limiting + +### Features + +* update libp2p deps ([#453](https://github.com/ipfs/js-ipfs-bitswap/issues/453)) ([2383088](https://github.com/ipfs/js-ipfs-bitswap/commit/2383088d6732ddae1cf3fe3470a832ab33013aad)) + +## [11.0.4](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.3...v11.0.4) (2022-06-24) + + +### Trivial Changes + +* remove unused dev dep ([#451](https://github.com/ipfs/js-ipfs-bitswap/issues/451)) ([34b19e0](https://github.com/ipfs/js-ipfs-bitswap/commit/34b19e0128d781a89742ca637388367638aefc63)) + +## [11.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.2...v11.0.3) (2022-06-23) + + +### Bug Fixes + +* iterate over connections instead of every peer in the peerstore ([#450](https://github.com/ipfs/js-ipfs-bitswap/issues/450)) ([dc9b126](https://github.com/ipfs/js-ipfs-bitswap/commit/dc9b12616a7c909063a23a8dcd71a08a379967bd)) + +### [11.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.1...v11.0.2) (2022-05-25) + + +### Trivial Changes + +* update libp2p interfaces ([#435](https://github.com/ipfs/js-ipfs-bitswap/issues/435)) ([c37ba88](https://github.com/ipfs/js-ipfs-bitswap/commit/c37ba88626bfb540844029f526c6f001f6667526)) + +### [11.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.0...v11.0.1) (2022-04-11) + + +### Bug Fixes + +* include dist folder in published package ([#430](https://github.com/ipfs/js-ipfs-bitswap/issues/430)) ([52d5fcc](https://github.com/ipfs/js-ipfs-bitswap/commit/52d5fcc3caeec6a1592e1bb422ae5afcbcab673d)) + +## [11.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v10.0.2...v11.0.0) (2022-04-07) + + +### ⚠ BREAKING CHANGES + +* this module is now ESM only + +### Features + +* update to typescript version of libp2p ([#428](https://github.com/ipfs/js-ipfs-bitswap/issues/428)) ([23d24ce](https://github.com/ipfs/js-ipfs-bitswap/commit/23d24ce295b90cd3fcb5b229b23258e6ff45d5c9)) + +### [10.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v10.0.1...v10.0.2) (2022-01-20) + + +### Bug Fixes + +* remove abort controller deps ([#402](https://github.com/ipfs/js-ipfs-bitswap/issues/402)) ([ef6c8ce](https://github.com/ipfs/js-ipfs-bitswap/commit/ef6c8ce03f53b27a4b64a4e7a3ab78f11a4dbccb)) + +### [10.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v10.0.0...v10.0.1) (2022-01-20) + + +### Bug Fixes + +* await network start and stop ([#415](https://github.com/ipfs/js-ipfs-bitswap/issues/415)) ([44dfcb5](https://github.com/ipfs/js-ipfs-bitswap/commit/44dfcb5dc40ed6ba8d3e4a5587d38634a37730c0)) + + +### Trivial Changes + +* switch to unified ci ([#408](https://github.com/ipfs/js-ipfs-bitswap/issues/408)) ([2d8e20f](https://github.com/ipfs/js-ipfs-bitswap/commit/2d8e20f6053c800f801060d042e3367dbbb97102)) + +# [10.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v9.0.0...v10.0.0) (2022-01-18) + + +### Features + +* libp2p async peerstore ([#413](https://github.com/ipfs/js-ipfs-bitswap/issues/413)) ([9f5fda9](https://github.com/ipfs/js-ipfs-bitswap/commit/9f5fda97903c4ffcd39d7affa887648df1169be3)) + + +### BREAKING CHANGES + +* peerstore methods are now all async + + + +# [9.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v8.0.0...v9.0.0) (2021-12-02) + + +### chore + +* update libp2p-interfaces ([#388](https://github.com/ipfs/js-ipfs-bitswap/issues/388)) ([c1bcae7](https://github.com/ipfs/js-ipfs-bitswap/commit/c1bcae752b48d3714f4ad696b9e6573b5ab4efa7)) + + +### BREAKING CHANGES + +* requires node 15+ + + + +# [8.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v7.0.1...v8.0.0) (2021-11-22) + + +### Features + +* update dht ([#384](https://github.com/ipfs/js-ipfs-bitswap/issues/384)) ([8d96a18](https://github.com/ipfs/js-ipfs-bitswap/commit/8d96a180a632a38b6470f696d2c0648dc5146dc5)) + + +### BREAKING CHANGES + +* uses 0.26.x of libp2p-kad-dht + + + +## [7.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v7.0.0...v7.0.1) (2021-11-19) + + +### Bug Fixes + +* encode cidv1 prefixes ([#383](https://github.com/ipfs/js-ipfs-bitswap/issues/383)) ([35be758](https://github.com/ipfs/js-ipfs-bitswap/commit/35be758dd1f3e5851ce23754bd12933783c16416)) + + + +# [7.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v6.0.2...v7.0.0) (2021-09-14) + + + +## [6.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v6.0.1...v6.0.2) (2021-09-10) + + +### chore + +* switch to esm ([#370](https://github.com/ipfs/js-ipfs-bitswap/issues/370)) ([63edf01](https://github.com/ipfs/js-ipfs-bitswap/commit/63edf01d1c3f15c049d48bfe7fda4b6efe3a3206)) + + +### BREAKING CHANGES + +* uses named exports only + + + +## [6.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v6.0.0...v6.0.1) (2021-08-23) + + + +# [6.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.6...v6.0.0) (2021-07-10) + + +### chore + +* update to new multiformats ([#340](https://github.com/ipfs/js-ipfs-bitswap/issues/340)) ([73bdd19](https://github.com/ipfs/js-ipfs-bitswap/commit/73bdd19dbe7e9c6ef557918f843ccdef92c859de)) + + +### BREAKING CHANGES + +* uses the CID class from the new multiformats module + + + +## [5.0.6](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.5...v5.0.6) (2021-06-22) + + + +## [5.0.5](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.4...v5.0.5) (2021-05-13) + + +### Bug Fixes + +* fixes unhandled promise rejection ([#337](https://github.com/ipfs/js-ipfs-bitswap/issues/337)) ([f41fd0b](https://github.com/ipfs/js-ipfs-bitswap/commit/f41fd0b4a60a945f71ac0ba3c2c1df659f4b3339)), closes [#332](https://github.com/ipfs/js-ipfs-bitswap/issues/332) + + + +## [5.0.4](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.3...v5.0.4) (2021-04-30) + + + +## [5.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.2...v5.0.3) (2021-04-20) + + +### Bug Fixes + +* specify pbjs root ([#323](https://github.com/ipfs/js-ipfs-bitswap/issues/323)) ([2bf0c2e](https://github.com/ipfs/js-ipfs-bitswap/commit/2bf0c2e51cb5ee63e88868e84ae67b4e3ee0ce9b)) + + + +## [5.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.1...v5.0.2) (2021-04-16) + + +### Bug Fixes + +* fix wrong type signature ([#304](https://github.com/ipfs/js-ipfs-bitswap/issues/304)) ([47fdb2a](https://github.com/ipfs/js-ipfs-bitswap/commit/47fdb2a8f8fc6142e9879869402401a65b04cb0a)) + + + +## [5.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.0...v5.0.1) (2021-03-10) + + +### Bug Fixes + +* fixes bignumber import for type gen ([#301](https://github.com/ipfs/js-ipfs-bitswap/issues/301)) ([5c09a2e](https://github.com/ipfs/js-ipfs-bitswap/commit/5c09a2ee20f438e33da71a061e662bfae3701c9d)) + + + +# [5.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v4.0.2...v5.0.0) (2021-03-09) + + +### Features + +* typedef generation & type checking ([#261](https://github.com/ipfs/js-ipfs-bitswap/issues/261)) ([fca78c8](https://github.com/ipfs/js-ipfs-bitswap/commit/fca78c8c501a92a9726eea0d5e6942cdd6cba983)) + + + +## [4.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v4.0.1...v4.0.2) (2021-01-29) + + + +## [4.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v4.0.0...v4.0.1) (2021-01-21) + + +### Bug Fixes + +* update provider multiaddrs before dial ([#286](https://github.com/ipfs/js-ipfs-bitswap/issues/286)) ([49cc66c](https://github.com/ipfs/js-ipfs-bitswap/commit/49cc66cf387a27c146f8f0a111c3dff90101f47a)) + + + +# [4.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v3.0.0...v4.0.0) (2020-11-06) + + + + +# [3.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v2.0.1...v3.0.0) (2020-08-24) + + +### Bug Fixes + +* replace node buffers with uint8arrays ([#251](https://github.com/ipfs/js-ipfs-bitswap/issues/251)) ([4f9d7cd](https://github.com/ipfs/js-ipfs-bitswap/commit/4f9d7cd)) + + +### BREAKING CHANGES + +* - All use of node Buffers have been replaced with Uint8Arrays +- All deps now use Uint8Arrays in place of node Buffers + + + + +## [2.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v2.0.0...v2.0.1) (2020-07-20) + + +### Bug Fixes + +* pass peer id to onPeerConnect ([#234](https://github.com/ipfs/js-ipfs-bitswap/issues/234)) ([bf3bf0c](https://github.com/ipfs/js-ipfs-bitswap/commit/bf3bf0c)), closes [ipfs/js-ipfs#3182](https://github.com/ipfs/js-ipfs/issues/3182) + + + + +# [2.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v1.0.0...v2.0.0) (2020-06-05) + + +### Features + +* use libp2p 0.28.x ([#217](https://github.com/ipfs/js-ipfs-bitswap/issues/217)) ([c4ede4d](https://github.com/ipfs/js-ipfs-bitswap/commit/c4ede4d)) + + +### BREAKING CHANGES + +* Requires `libp2p@0.28.x` or above + +Co-authored-by: Jacob Heun + + + + +# [1.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.29.2...v1.0.0) (2020-05-27) + + +### Bug Fixes + +* do not rebroadcast want list ([#225](https://github.com/ipfs/js-ipfs-bitswap/issues/225)) ([313ae3b](https://github.com/ipfs/js-ipfs-bitswap/commit/313ae3b)), closes [#160](https://github.com/ipfs/js-ipfs-bitswap/issues/160) +* race condition when requesting the same block twice ([#214](https://github.com/ipfs/js-ipfs-bitswap/issues/214)) ([78ce032](https://github.com/ipfs/js-ipfs-bitswap/commit/78ce032)) + + +### Performance Improvements + +* decrease wantlist send debounce time ([#224](https://github.com/ipfs/js-ipfs-bitswap/issues/224)) ([46490f5](https://github.com/ipfs/js-ipfs-bitswap/commit/46490f5)) + + + + +## [0.29.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.29.1...v0.29.2) (2020-05-07) + + +### Bug Fixes + +* re-sort queue after adding tasks ([#221](https://github.com/ipfs/js-ipfs-bitswap/issues/221)) ([1a5ed4a](https://github.com/ipfs/js-ipfs-bitswap/commit/1a5ed4a)), closes [ipfs/js-ipfs#2992](https://github.com/ipfs/js-ipfs/issues/2992) +* survive bad network requests ([#222](https://github.com/ipfs/js-ipfs-bitswap/issues/222)) ([2fc7023](https://github.com/ipfs/js-ipfs-bitswap/commit/2fc7023)), closes [#221](https://github.com/ipfs/js-ipfs-bitswap/issues/221) +* **ci:** add empty commit to fix lint checks on master ([7872a19](https://github.com/ipfs/js-ipfs-bitswap/commit/7872a19)) + + + + +## [0.29.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.29.0...v0.29.1) (2020-04-27) + + +### Bug Fixes + +* really remove node globals ([#219](https://github.com/ipfs/js-ipfs-bitswap/issues/219)) ([120d1c7](https://github.com/ipfs/js-ipfs-bitswap/commit/120d1c7)) + + + + +# [0.29.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.28.0...v0.29.0) (2020-04-23) + + +### Bug Fixes + +* use ipld-block and remove node globals ([#218](https://github.com/ipfs/js-ipfs-bitswap/issues/218)) ([6b4dc32](https://github.com/ipfs/js-ipfs-bitswap/commit/6b4dc32)) + + +### BREAKING CHANGES + +* swaps ipfs-block with ipld-block + +related to https://github.com/ipfs/js-ipfs/issues/2924 + + + + +# [0.28.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.27.1...v0.28.0) (2020-04-09) + + + + +## [0.27.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.27.0...v0.27.1) (2020-02-10) + + +### Bug Fixes + +* await result of receiving blocks ([#213](https://github.com/ipfs/js-ipfs-bitswap/issues/213)) ([dae48dd](https://github.com/ipfs/js-ipfs-bitswap/commit/dae48dd)) + + + + +# [0.27.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.26.2...v0.27.0) (2020-01-28) + + + + +## [0.26.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.26.1...v0.26.2) (2019-12-22) + + +### Bug Fixes + +* use multicodec correctly ([#209](https://github.com/ipfs/js-ipfs-bitswap/issues/209)) ([579ddb5](https://github.com/ipfs/js-ipfs-bitswap/commit/579ddb5)) + + + + +## [0.26.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.26.0...v0.26.1) (2019-12-11) + + +### Bug Fixes + +* reduce size ([#203](https://github.com/ipfs/js-ipfs-bitswap/issues/203)) ([9f818b4](https://github.com/ipfs/js-ipfs-bitswap/commit/9f818b4)) + + + + +# [0.26.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.25.1...v0.26.0) (2019-09-24) + + +### Code Refactoring + +* callbacks -> async / await ([#202](https://github.com/ipfs/js-ipfs-bitswap/issues/202)) ([accf53b](https://github.com/ipfs/js-ipfs-bitswap/commit/accf53b)) + + +### BREAKING CHANGES + +* All places in the API that used callbacks are now replaced with async/await + +* feat: make `get()` a generator + +* make `getMany()` AsyncIterable + +* feat: make `put()` a generator + +* make `putMany()` AsyncIterable + +* remove check in `_findAndConnect()` + +* feat: make `start()` and `stop()` async/await + +* refactor: make `connectTo()` async/await + +* refactor: make `findProviders()` and `findAndConnect()` async/await + +* refactor: cb => async + +* refactor: async/await + +* chore: update travis + +* refactor: update benchmark tests and allow streaming to putMany + +* chore: address pr comments + +* chore: remove callback hell eslint disables + +* chore: wrap list of tasks in promise.all + +* chore: callbackify methods inside pull stream + +* chore: accept PR suggestions + +* chore: fix typo + + + + +## [0.25.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.25.0...v0.25.1) (2019-06-26) + + +### Bug Fixes + +* use consistent encoding for cid comparison ([c8cee6a](https://github.com/ipfs/js-ipfs-bitswap/commit/c8cee6a)) + + +### BREAKING CHANGES + +* Emitted events have different bytes + +The emitted events contain the stringified version of the CID, as we +change it to the base encoding the CID has, those bytes may be different +to previous versions of this module. + +Though this shouldn't have any impact on any other modules as the +events are only used internally. + + + + +# [0.25.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.24.1...v0.25.0) (2019-06-12) + + +### Bug Fixes + +* base encode CIDs before logging or emitting them ([704de22](https://github.com/ipfs/js-ipfs-bitswap/commit/704de22)) + + +### BREAKING CHANGES + +* Emitted events have different bytes + +The emitted events contain the stringified version of the CID, as we +change it to the base encoding the CID has, those bytes may be different +to previous versions of this module. + +Though this shouldn't have any impact on any other modules as the +events are only used internally. + + + + +## [0.24.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.24.0...v0.24.1) (2019-05-30) + + +### Bug Fixes + +* ignore unwanted blocks ([#194](https://github.com/ipfs/js-ipfs-bitswap/issues/194)) ([e8d722c](https://github.com/ipfs/js-ipfs-bitswap/commit/e8d722c)) + + + + +# [0.24.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.23.0...v0.24.0) (2019-05-09) + + +### Chores + +* update cids dependency ([0779160](https://github.com/ipfs/js-ipfs-bitswap/commit/0779160)) + + +### BREAKING CHANGES + +* v1 CIDs created by this module now default to base32 encoding when stringified + +refs: https://github.com/ipfs/js-ipfs/issues/1995 + +License: MIT +Signed-off-by: Alan Shaw + + + + +# [0.23.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.22.0...v0.23.0) (2019-03-16) + + + + +# [0.22.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.21.2...v0.22.0) (2019-01-08) + + +### Bug Fixes + +* reduce bundle size ([d8f8040](https://github.com/ipfs/js-ipfs-bitswap/commit/d8f8040)) + + +### BREAKING CHANGES + +* change from big.js to bignumber.js + +The impact of this change is only on the `snapshot` field of +the stats, as those values are represented as Big Numbers. + + + + +## [0.21.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.21.1...v0.21.2) (2019-01-08) + + +### Bug Fixes + +* avoid sync callbacks in async code ([ddfdd71](https://github.com/ipfs/js-ipfs-bitswap/commit/ddfdd71)) +* ensure callback is called ([c27318f](https://github.com/ipfs/js-ipfs-bitswap/commit/c27318f)) + + + + +## [0.21.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.21.0...v0.21.1) (2018-12-06) + + +### Features + +* send max providers to findProviders request ([31493dc](https://github.com/ipfs/js-ipfs-bitswap/commit/31493dc)) + + + + +# [0.21.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.20.3...v0.21.0) (2018-10-26) + + +### Features + +* change bitswapLedgerForPeer output format ([c68a0c8](https://github.com/ipfs/js-ipfs-bitswap/commit/c68a0c8)) + + + + +## [0.20.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.20.2...v0.20.3) (2018-07-03) + + + + +## [0.20.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.20.0...v0.20.2) (2018-06-18) + + +### Bug Fixes + +* ipfs/js-ipfs[#1292](https://github.com/ipfs/js-ipfs-bitswap/issues/1292) - Catch invalid CIDs and return the error via callback ([#170](https://github.com/ipfs/js-ipfs-bitswap/issues/170)) ([51f5ce0](https://github.com/ipfs/js-ipfs-bitswap/commit/51f5ce0)) +* reset batch size counter ([739ad0d](https://github.com/ipfs/js-ipfs-bitswap/commit/739ad0d)) + + +### Features + +* add bitswap.ledgerForPeer ([871d0d2](https://github.com/ipfs/js-ipfs-bitswap/commit/871d0d2)) +* add ledger.debtRatio() ([e602810](https://github.com/ipfs/js-ipfs-bitswap/commit/e602810)) + + + + +## [0.20.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.20.0...v0.20.1) (2018-05-28) + + +### Bug Fixes + +* ipfs/js-ipfs[#1292](https://github.com/ipfs/js-ipfs-bitswap/issues/1292) - Catch invalid CIDs and return the error via callback ([#170](https://github.com/ipfs/js-ipfs-bitswap/issues/170)) ([51f5ce0](https://github.com/ipfs/js-ipfs-bitswap/commit/51f5ce0)) +* reset batch size counter ([739ad0d](https://github.com/ipfs/js-ipfs-bitswap/commit/739ad0d)) + + + + +# [0.20.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.19.0...v0.20.0) (2018-04-10) + + + + +# [0.19.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.18.1...v0.19.0) (2018-02-14) + + +### Features + +* update network calls to use dialProtocol instead ([b669aac](https://github.com/ipfs/js-ipfs-bitswap/commit/b669aac)) + + + + +## [0.18.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.18.0...v0.18.1) (2018-02-06) + + +### Bug Fixes + +* getMany: ensuring we set the want list ([#162](https://github.com/ipfs/js-ipfs-bitswap/issues/162)) ([8e91def](https://github.com/ipfs/js-ipfs-bitswap/commit/8e91def)) + + +### Features + +* added getMany performance tests ([#164](https://github.com/ipfs/js-ipfs-bitswap/issues/164)) ([b349085](https://github.com/ipfs/js-ipfs-bitswap/commit/b349085)) +* per-peer stats ([#166](https://github.com/ipfs/js-ipfs-bitswap/issues/166)) ([ff978d0](https://github.com/ipfs/js-ipfs-bitswap/commit/ff978d0)) + + + + +# [0.18.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.4...v0.18.0) (2017-12-15) + + +### Features + +* stats improvements ([#158](https://github.com/ipfs/js-ipfs-bitswap/issues/158)) ([17e15d0](https://github.com/ipfs/js-ipfs-bitswap/commit/17e15d0)) + + + + +## [0.17.4](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.3...v0.17.4) (2017-11-10) + + +### Features + +* windows interop ([#154](https://github.com/ipfs/js-ipfs-bitswap/issues/154)) ([a8b1e07](https://github.com/ipfs/js-ipfs-bitswap/commit/a8b1e07)) + + + + +## [0.17.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.2...v0.17.3) (2017-11-08) + + +### Bug Fixes + +* add missing multicodec dependency ([#155](https://github.com/ipfs/js-ipfs-bitswap/issues/155)) ([751d436](https://github.com/ipfs/js-ipfs-bitswap/commit/751d436)) + + + + +## [0.17.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.1...v0.17.2) (2017-09-07) + + + + +## [0.17.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.0...v0.17.1) (2017-09-07) + + +### Features + +* replace protocol-buffers with protons ([#149](https://github.com/ipfs/js-ipfs-bitswap/issues/149)) ([ca8fa72](https://github.com/ipfs/js-ipfs-bitswap/commit/ca8fa72)) + + + + +# [0.17.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.16.1...v0.17.0) (2017-09-03) diff --git a/packages/bitswap/LICENSE b/packages/bitswap/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/bitswap/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/bitswap/LICENSE-APACHE b/packages/bitswap/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/bitswap/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/bitswap/LICENSE-MIT b/packages/bitswap/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/bitswap/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/bitswap/README.md b/packages/bitswap/README.md new file mode 100644 index 00000000..fc110943 --- /dev/null +++ b/packages/bitswap/README.md @@ -0,0 +1,47 @@ +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain) + +> JavaScript implementation of the Bitswap data exchange protocol used by Helia + +# About + +This module implements the [Bitswap protocol](https://docs.ipfs.tech/concepts/bitswap/) in TypeScript. + +# Install + +```console +$ npm i @helia/bitswap +``` + +## Browser ` +``` + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/bitswap/package.json b/packages/bitswap/package.json new file mode 100644 index 00000000..3e9b1ef4 --- /dev/null +++ b/packages/bitswap/package.json @@ -0,0 +1,204 @@ +{ + "name": "@helia/bitswap", + "version": "0.0.0", + "description": "JavaScript implementation of the Bitswap data exchange protocol used by Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia/tree/main/packages/bitswap#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia/issues" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "keywords": [ + "exchange", + "ipfs", + "libp2p", + "p2p" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "project": true, + "sourceType": "module" + }, + "ignorePatterns": [ + "scripts/*", + "*.test-d.ts" + ] + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test", + "test:node": "aegir test -t node --cov", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check -i protons -i events", + "generate": "protons ./src/pb/message.proto", + "docs": "aegir docs" + }, + "dependencies": { + "@helia/interface": "^3.0.1", + "@libp2p/interface": "^1.1.1", + "@libp2p/logger": "^4.0.4", + "@libp2p/peer-collections": "^5.1.5", + "@libp2p/utils": "^5.2.0", + "@multiformats/multiaddr": "^12.1.0", + "@multiformats/multiaddr-matcher": "^1.1.2", + "any-signal": "^4.1.1", + "debug": "^4.3.4", + "events": "^3.3.0", + "interface-blockstore": "^5.2.7", + "interface-store": "^5.1.5", + "it-all": "^3.0.4", + "it-drain": "^3.0.5", + "it-filter": "^3.0.4", + "it-length-prefixed": "^9.0.0", + "it-length-prefixed-stream": "^1.1.6", + "it-map": "^3.0.5", + "it-pipe": "^3.0.1", + "it-take": "^3.0.1", + "multiformats": "^13.0.0", + "p-defer": "^4.0.0", + "progress-events": "^1.0.0", + "protons-runtime": "^5.0.0", + "race-signal": "^1.0.2", + "uint8-varint": "^2.0.3", + "uint8arraylist": "^2.4.3", + "uint8arrays": "^5.0.1" + }, + "devDependencies": { + "@libp2p/interface-compliance-tests": "^5.1.3", + "@libp2p/peer-id-factory": "^4.0.5", + "@types/lodash.difference": "^4.5.7", + "@types/lodash.flatten": "^4.4.7", + "@types/lodash.range": "^3.2.7", + "@types/sinon": "^17.0.3", + "@types/stats-lite": "^2.2.0", + "@types/varint": "^6.0.0", + "aegir": "^42.1.0", + "blockstore-core": "^4.3.10", + "delay": "^6.0.0", + "it-pair": "^2.0.6", + "it-protobuf-stream": "^1.1.2", + "p-event": "^6.0.0", + "p-retry": "^6.2.0", + "protons": "^7.0.2", + "sinon": "^17.0.1", + "sinon-ts": "^2.0.0" + }, + "browser": { + "dist/test/utils/create-libp2p-node.js": false + }, + "sideEffects": false +} diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts new file mode 100644 index 00000000..85b0c15f --- /dev/null +++ b/packages/bitswap/src/bitswap.ts @@ -0,0 +1,481 @@ +/* eslint-disable no-loop-func */ +import { setMaxListeners } from '@libp2p/interface' +import { PeerSet } from '@libp2p/peer-collections' +import { PeerQueue } from '@libp2p/utils/peer-queue' +import { anySignal } from 'any-signal' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' +import pDefer from 'p-defer' +import { CodeError } from 'protons-runtime' +import { raceSignal } from 'race-signal' +import { DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY, DEFAULT_SESSION_QUERY_CONCURRENCY, DEFAULT_SESSION_ROOT_PRIORITY } from './constants.js' +import { Network } from './network.js' +import { Notifications, receivedBlockEvent, type ReceivedBlockListener, type HaveBlockListener, haveEvent, type DoNotHaveBlockListener, doNotHaveEvent } from './notifications.js' +import { BlockPresenceType, WantType } from './pb/message.js' +import { PeerWantLists } from './peer-want-lists/index.js' +import { createBitswapSession } from './session.js' +import { Stats } from './stats.js' +import vd from './utils/varint-decoder.js' +import { WantList } from './want-list.js' +import type { BitswapOptions, Bitswap as BitswapInterface, MultihashHasherLoader, BitswapWantProgressEvents, BitswapNotifyProgressEvents, BitswapSession, WantListEntry, CreateSessionOptions, BitswapComponents } from './index.js' +import type { BitswapMessage } from './pb/message.js' +import type { ComponentLogger, Libp2p, PeerId } from '@libp2p/interface' +import type { Logger } from '@libp2p/logger' +import type { AbortOptions } from '@multiformats/multiaddr' +import type { Blockstore } from 'interface-blockstore' +import type { ProgressOptions } from 'progress-events' + +export interface WantOptions extends AbortOptions, ProgressOptions { + /** + * When searching the routing for providers, stop searching after finding this + * many providers. + * + * @default 3 + */ + maxProviders?: number +} + +/** + * JavaScript implementation of the Bitswap 'data exchange' protocol + * used by IPFS. + */ +export class Bitswap implements BitswapInterface { + private readonly log: Logger + private readonly logger: ComponentLogger + public readonly stats: Stats + public network: Network + public blockstore: Blockstore + public peerWantLists: PeerWantLists + public wantList: WantList + public notifications: Notifications + public status: 'starting' | 'started' | 'stopping' | 'stopped' + private readonly hashLoader?: MultihashHasherLoader + + private readonly libp2p: Libp2p + + constructor (components: BitswapComponents, init: BitswapOptions = {}) { + this.logger = components.logger + this.log = components.logger.forComponent('bitswap') + this.status = 'stopped' + this.libp2p = components.libp2p + this.blockstore = components.blockstore + this.hashLoader = init.hashLoader + + // report stats to libp2p metrics + this.stats = new Stats(components) + + // the network delivers messages + this.network = new Network(components, init) + this.network.addEventListener('bitswap:message', evt => { + this._receiveMessage(evt.detail.peer, evt.detail.message) + .catch(err => { + this.log.error('error receiving bitswap message from %p', evt.detail.peer, err) + }) + }) + this.network.addEventListener('peer:connected', evt => { + this.wantList.connected(evt.detail) + .catch(err => { + this.log.error('error processing newly connected bitswap peer %p', evt.detail, err) + }) + }) + this.network.addEventListener('peer:disconnected', evt => { + this.wantList.disconnected(evt.detail) + this.peerWantLists.peerDisconnected(evt.detail) + }) + + // handle which blocks we send to peers + this.peerWantLists = new PeerWantLists({ + ...components, + network: this.network + }, init) + + // handle which blocks we ask peers for + this.wantList = new WantList({ + ...components, + network: this.network + }) + + // event emitter that lets sessions/want promises know blocks have arrived + this.notifications = new Notifications(components) + } + + /** + * handle messages received through the network + */ + async _receiveMessage (peerId: PeerId, message: BitswapMessage): Promise { + // hash all incoming blocks + const received = await Promise.all( + message.blocks + .filter(block => block.prefix != null && block.data != null) + .map(async block => { + const values = vd(block.prefix) + const cidVersion = values[0] + const multicodec = values[1] + const hashAlg = values[2] + // const hashLen = values[3] // We haven't need to use this so far + + const hasher = hashAlg === sha256.code ? sha256 : await this.hashLoader?.getHasher(hashAlg) + + if (hasher == null) { + throw new CodeError('Unknown hash algorithm', 'ERR_UNKNOWN_HASH_ALG') + } + + const hash = await hasher.digest(block.data) + const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash) + const wasWanted = this.notifications.listenerCount(receivedBlockEvent(cid)) > 0 + + return { wasWanted, cid, data: block.data } + }) + ) + + // quickly send out cancels, reduces chances of duplicate block receives + if (received.length > 0) { + this.wantList.cancelWants( + received + .filter(({ wasWanted }) => wasWanted) + .map(({ cid }) => cid) + ).catch(err => { + this.log.error('error sending block cancels', err) + }) + } + + // notify sessions of block haves/don't haves + for (const { cid: cidBytes, type } of message.blockPresences) { + const cid = CID.decode(cidBytes) + + if (type === BlockPresenceType.HaveBlock) { + this.notifications.haveBlock(cid, peerId) + } else { + this.notifications.doNotHaveBlock(cid, peerId) + } + } + + await Promise.all( + received.map( + async ({ cid, wasWanted, data }) => { + await this._handleReceivedBlock(peerId, cid, data, wasWanted) + this.notifications.receivedBlock(cid, data, peerId) + } + ) + ) + + try { + // Note: this allows the engine to respond to any wants in the message. + // Processing of the blocks in the message happens below, after the + // blocks have been added to the blockstore. + await this.peerWantLists.messageReceived(peerId, message) + } catch (err) { + // Log instead of throwing an error so as to process as much as + // possible of the message. Currently `messageReceived` does not + // throw any errors, but this could change in the future. + this.log('failed to receive message from %p', peerId, message) + } + } + + private async _handleReceivedBlock (peerId: PeerId, cid: CID, data: Uint8Array, wasWanted: boolean): Promise { + this.log('received block') + + const has = await this.blockstore.has(cid) + + this._updateReceiveCounters(peerId, cid, data, has) + + if (!wasWanted || has) { + return + } + + await this.blockstore.put(cid, data) + } + + _updateReceiveCounters (peerId: PeerId, cid: CID, data: Uint8Array, exists: boolean): void { + this.stats.updateBlocksReceived(1, peerId) + this.stats.updateDataReceived(data.byteLength, peerId) + + if (exists) { + this.stats.updateDuplicateBlocksReceived(1, peerId) + this.stats.updateDuplicateDataReceived(data.byteLength, peerId) + } + } + + async createSession (root: CID, options?: CreateSessionOptions): Promise { + const minProviders = options?.minProviders ?? DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY + const maxProviders = options?.maxProviders ?? DEFAULT_MAX_PROVIDERS_PER_REQUEST + + // normalize to v1 CID + root = root.toV1() + + const deferred = pDefer() + const session = createBitswapSession({ + notifications: this.notifications, + wantList: this.wantList, + network: this.network, + logger: this.logger + }, { + root + }) + + let peerDoesNotHave = 0 + let searchedForProviders = false + + const queue = new PeerQueue({ + concurrency: options?.queryConcurrency ?? DEFAULT_SESSION_QUERY_CONCURRENCY + }) + queue.addEventListener('error', (err) => { + this.log.error('error querying peer for %c', root, err) + }) + queue.addEventListener('completed', () => { + if (session.peers.size === maxProviders) { + this.notifications.removeListener(receivedBlockEvent(root), receivedBlockListener) + this.notifications.removeListener(haveEvent(root), haveBlockListener) + this.notifications.removeListener(doNotHaveEvent(root), doNotHaveBlockListener) + } + + queue.clear() + }) + + const queriedPeers = new PeerSet() + const existingPeers = new PeerSet() + const providerPeers = new PeerSet() + + // register for peer responses + const receivedBlockListener: ReceivedBlockListener = (block, peer): void => { + this.log('adding %p to session after receiving block when asking for HAVE_BLOCK', peer) + session.peers.add(peer) + + // check if the session can be used now + if (session.peers.size === minProviders) { + deferred.resolve(session) + } + } + const haveBlockListener: HaveBlockListener = (peer): void => { + this.log('adding %p to session after receiving HAVE_BLOCK', peer) + session.peers.add(peer) + + // check if the session can be used now + if (session.peers.size === minProviders) { + deferred.resolve(session) + } + + if (session.peers.size === maxProviders) { + this.notifications.removeListener(receivedBlockEvent(root), receivedBlockListener) + this.notifications.removeListener(haveEvent(root), haveBlockListener) + this.notifications.removeListener(doNotHaveEvent(root), doNotHaveBlockListener) + } + } + const doNotHaveBlockListener: DoNotHaveBlockListener = (peer) => { + peerDoesNotHave++ + + if (searchedForProviders && peerDoesNotHave === queriedPeers.size) { + // no queried peers can supply the root block + deferred.reject(new CodeError(`No peers or providers had ${root}`, 'ERR_NO_PROVIDERS_FOUND')) + } + } + + this.notifications.addListener(receivedBlockEvent(root), receivedBlockListener) + this.notifications.addListener(haveEvent(root), haveBlockListener) + this.notifications.addListener(doNotHaveEvent(root), doNotHaveBlockListener) + + if (options?.queryConnectedPeers !== false) { + // ask our current bitswap peers for the CID + await Promise.all([ + ...this.wantList.peers.keys() + ].map(async (peerId) => { + if (queriedPeers.has(peerId)) { + return + } + + existingPeers.add(peerId) + + await queue.add(async () => { + try { + await this.network.sendMessage(peerId, { + wantlist: { + entries: [{ + cid: root.bytes, + priority: options?.priority ?? DEFAULT_SESSION_ROOT_PRIORITY, + wantType: WantType.WantHave, + sendDontHave: true + }] + } + }, options) + + queriedPeers.add(peerId) + } catch (err: any) { + this.log.error('error querying connected peer %p for initial session', peerId, err) + } + }, { + peerId + }) + })) + + this.log.trace('creating session queried %d connected peers for %c', queriedPeers, root) + } + + // find network providers too but do not wait for the query to complete + void Promise.resolve().then(async () => { + let providers = 0 + + for await (const provider of this.network.findProviders(root, options)) { + providers++ + + if (queriedPeers.has(provider.id)) { + continue + } + + await queue.add(async () => { + try { + await this.network.sendMessage(provider.id, { + wantlist: { + entries: [{ + cid: root.bytes, + priority: options?.priority ?? DEFAULT_SESSION_ROOT_PRIORITY, + wantType: WantType.WantHave, + sendDontHave: true + }] + } + }, options) + + providerPeers.add(provider.id) + queriedPeers.add(provider.id) + } catch (err: any) { + this.log.error('error querying provider %p for initial session', provider.id, err.errors ?? err) + } + }, { + peerId: provider.id + }) + + if (session.peers.size === maxProviders) { + break + } + } + + this.log.trace('creating session found %d providers for %c', providers, root) + + searchedForProviders = true + + // we have no peers and could find no providers + if (providers === 0) { + deferred.reject(new CodeError(`Could not find providers for ${root}`, 'ERR_NO_PROVIDERS_FOUND')) + } + }) + .catch(err => { + this.log.error('error querying providers for %c', root, err) + }) + .finally(() => { + if (peerDoesNotHave === queriedPeers.size) { + // no queried peers can supply the root block + deferred.reject(new CodeError(`No peers or providers had ${root}`, 'ERR_NO_PROVIDERS_FOUND')) + } + }) + + return raceSignal(deferred.promise, options?.signal) + } + + async want (cid: CID, options: WantOptions = {}): Promise { + const loadOrFetchFromNetwork = async (cid: CID, wantBlockPromise: Promise, options: WantOptions): Promise => { + try { + // have to await here as we want to handle ERR_NOT_FOUND from the + // blockstore + return await Promise.race([ + this.blockstore.get(cid, options), + wantBlockPromise + ]) + } catch (err: any) { + if (err.code !== 'ERR_NOT_FOUND') { + throw err + } + + // add the block to the wantlist + this.wantList.wantBlocks([cid]) + .catch(err => { + this.log.error('error adding %c to wantlist', cid, err) + }) + + // find providers and connect to them + this.network.findAndConnect(cid, options) + .catch(err => { + this.log.error('could not find and connect for cid %c', cid, err) + }) + + return wantBlockPromise + } + } + + // it's possible for blocks to come in while we do the async operations to + // get them from the blockstore leading to a race condition, so register for + // incoming block notifications as well as trying to get it from the + // datastore + const controller = new AbortController() + setMaxListeners(Infinity, controller.signal) + const signal = anySignal([controller.signal, options.signal]) + + try { + const wantBlockPromise = this.notifications.wantBlock(cid, { + ...options, + signal + }) + + const block = await Promise.race([ + wantBlockPromise, + loadOrFetchFromNetwork(cid, wantBlockPromise, { + ...options, + signal + }) + ]) + + return block + } catch (err: any) { + if (err.code === 'ERR_ABORTED') { + // the want was cancelled, send out cancel messages + await this.wantList.cancelWants([cid]) + } + + throw err + } finally { + // since we have the block we can now abort any outstanding attempts to + // fetch it + controller.abort() + signal.clear() + + this.wantList.unwantBlocks([cid]) + } + } + + /** + * Sends notifications about the arrival of a block + */ + async notify (cid: CID, block: Uint8Array, options: ProgressOptions & AbortOptions = {}): Promise { + this.notifications.receivedBlock(cid, block, this.libp2p.peerId) + + await this.peerWantLists.receivedBlock(cid, options) + } + + getWantlist (): WantListEntry[] { + return [...this.wantList.wants.values()] + } + + getPeerWantlist (peer: PeerId): WantListEntry[] | undefined { + return this.peerWantLists.wantListForPeer(peer) + } + + /** + * Start the bitswap node + */ + async start (): Promise { + this.status = 'starting' + + this.wantList.start() + await this.network.start() + this.status = 'started' + } + + /** + * Stop the bitswap node + */ + async stop (): Promise { + this.status = 'stopping' + + this.wantList.stop() + await this.network.stop() + this.status = 'stopped' + } +} diff --git a/packages/bitswap/src/constants.ts b/packages/bitswap/src/constants.ts new file mode 100644 index 00000000..ae8c86d3 --- /dev/null +++ b/packages/bitswap/src/constants.ts @@ -0,0 +1,12 @@ +export const BITSWAP_120 = '/ipfs/bitswap/1.2.0' +export const DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY = 1 +export const DEFAULT_MAX_PROVIDERS_PER_REQUEST = 3 +export const DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK = 1024 +export const DEFAULT_MAX_INBOUND_STREAMS = 1024 +export const DEFAULT_MAX_OUTBOUND_STREAMS = 1024 +export const DEFAULT_MESSAGE_RECEIVE_TIMEOUT = 5000 +export const DEFAULT_MESSAGE_SEND_TIMEOUT = 5000 +export const DEFAULT_MESSAGE_SEND_CONCURRENCY = 50 +export const DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS = false +export const DEFAULT_SESSION_ROOT_PRIORITY = 1 +export const DEFAULT_SESSION_QUERY_CONCURRENCY = 5 diff --git a/packages/bitswap/src/index.ts b/packages/bitswap/src/index.ts new file mode 100644 index 00000000..e37004b1 --- /dev/null +++ b/packages/bitswap/src/index.ts @@ -0,0 +1,236 @@ +/** + * @packageDocumentation + * + * This module implements the [Bitswap protocol](https://docs.ipfs.tech/concepts/bitswap/) in TypeScript. + */ + +import { Bitswap as BitswapClass } from './bitswap.js' +import type { BitswapNetworkNotifyProgressEvents, BitswapNetworkWantProgressEvents } from './network.js' +import type { WantType } from './pb/message.js' +import type { Routing } from '@helia/interface' +import type { Libp2p, AbortOptions, Startable, ComponentLogger, Metrics, PeerId } from '@libp2p/interface' +import type { PeerSet } from '@libp2p/peer-collections' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' +import type { MultihashHasher } from 'multiformats/hashes/interface' +import type { ProgressEvent, ProgressOptions } from 'progress-events' + +export type BitswapWantProgressEvents = + BitswapWantBlockProgressEvents + +export type BitswapNotifyProgressEvents = + BitswapNetworkNotifyProgressEvents + +export type BitswapWantBlockProgressEvents = + ProgressEvent<'bitswap:want-block:unwant', CID> | + ProgressEvent<'bitswap:want-block:block', CID> | + BitswapNetworkWantProgressEvents + +/** + * A bitswap session is a network overlay consisting of peers that all have the + * first block in a file. Subsequent requests will only go to these peers. + */ +export interface BitswapSession { + /** + * The peers in this session + */ + peers: PeerSet + + /** + * Fetch an additional CID from this DAG + */ + want(cid: CID, options?: AbortOptions & ProgressOptions): Promise +} + +export interface WantListEntry { + cid: CID + session: PeerSet + priority: number + wantType: WantType + cancel: boolean + sendDontHave: boolean + + /** + * Whether we have sent the dont-have block presence + */ + sentDontHave?: boolean +} + +export interface CreateSessionOptions extends AbortOptions, ProgressOptions { + /** + * The session will be ready after this many providers for the root CID have + * been found. Providers will continue to be added to the session after this + * until they reach `maxProviders`. + * + * @default 1 + */ + minProviders?: number + + /** + * After this many providers for the root CID have been found, stop searching + * for more providers. + * + * @default 3 + */ + maxProviders?: number + + /** + * If true, query connected peers before searching for providers in the + * routing. + * + * @default true + */ + queryConnectedPeers?: boolean + + /** + * The priority to use when querying availability of the root CID + * + * @default 1 + */ + priority?: number + + /** + * How many peers/providers to send the initial query for the root CID to at + * the same time + * + * @default 5 + */ + queryConcurrency?: number +} + +export interface Bitswap extends Startable { + /** + * Returns the current state of the wantlist + */ + getWantlist(): WantListEntry[] + + /** + * Returns the current state of the wantlist for a peer, if it is being + * tracked + */ + getPeerWantlist(peerId: PeerId): WantListEntry[] | undefined + + /** + * Notify bitswap that a new block is available + */ + notify(cid: CID, block: Uint8Array, options?: ProgressOptions): Promise + + /** + * Start a session to retrieve a file from the network + */ + want(cid: CID, options?: AbortOptions & ProgressOptions): Promise + + /** + * Start a session to retrieve a file from the network + */ + createSession(root: CID, options?: AbortOptions & ProgressOptions): Promise +} + +export interface MultihashHasherLoader { + getHasher(codeOrName: number | string): Promise +} + +export interface BitswapComponents { + routing: Routing + blockstore: Blockstore + logger: ComponentLogger + libp2p: Libp2p + metrics?: Metrics +} + +export interface BitswapOptions { + /** + * This is the maximum number of concurrent inbound bitswap streams that are + * allowed + * + * @default 32 + */ + maxInboundStreams?: number + + /** + * This is the maximum number of concurrent outbound bitswap streams that are + * allowed + * + * @default 128 + */ + maxOutboundStreams?: number + + /** + * An incoming stream must resolve within this number of seconds + * + * @default 30000 + */ + incomingStreamTimeout?: number + + /** + * Whether to run on transient (e.g. time/data limited) connections + * + * @default false + */ + runOnTransientConnections?: boolean + + /** + * Enables loading esoteric hash functions + */ + hashLoader?: MultihashHasherLoader + + /** + * The protocol that we speak + * + * @default '/ipfs/bitswap/1.2.0' + */ + protocol?: string + + /** + * When a new peer connects, sending our WantList should complete within this + * many ms + * + * @default 5000 + */ + messageSendTimeout?: number + + /** + * When sending want list updates to peers, how many messages to send at once + * + * @default 50 + */ + messageSendConcurrency?: number + + /** + * When sending blocks to peers, how many messages to send at once + * + * @default 50 + */ + sendBlocksConcurrency?: number + + /** + * When sending want list updates to peers, how many messages to send at once + * + * @default 10000 + */ + sendBlocksTimeout?: number + + /** + * When a block is added to the blockstore and we are about to sending that + * block to peers who have it in their wantlist, wait this long before + * queueing the send job in case more blocks are added that they want + * + * @default 10 + */ + sendBlocksDebounce?: number + + /** + * If the client sends a want-have, and the engine has the corresponding + * block, we check the size of the block and if it's small enough we send the + * block itself, rather than sending a HAVE. + * + * This defines the maximum size up to which we replace a HAVE with a block. + * + * @default 1024 + */ + maxSizeReplaceHasWithBlock?: number +} + +export const createBitswap = (components: BitswapComponents, options: BitswapOptions = {}): Bitswap => { + return new BitswapClass(components, options) +} diff --git a/packages/bitswap/src/network.ts b/packages/bitswap/src/network.ts new file mode 100644 index 00000000..d1834f91 --- /dev/null +++ b/packages/bitswap/src/network.ts @@ -0,0 +1,476 @@ +import { CodeError, TypedEventEmitter, setMaxListeners } from '@libp2p/interface' +import { PeerQueue, type PeerQueueJobOptions } from '@libp2p/utils/peer-queue' +import { Circuit } from '@multiformats/multiaddr-matcher' +import { anySignal } from 'any-signal' +import debug from 'debug' +import drain from 'it-drain' +import * as lp from 'it-length-prefixed' +import { lpStream } from 'it-length-prefixed-stream' +import map from 'it-map' +import { pipe } from 'it-pipe' +import take from 'it-take' +import { base64 } from 'multiformats/bases/base64' +import { CID } from 'multiformats/cid' +import { CustomProgressEvent } from 'progress-events' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_TIMEOUT, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js' +import { BitswapMessage } from './pb/message.js' +import type { WantOptions } from './bitswap.js' +import type { MultihashHasherLoader } from './index.js' +import type { Block, BlockPresence, WantlistEntry } from './pb/message.js' +import type { Provider, Routing } from '@helia/interface' +import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, MetricGroup, ComponentLogger, Metrics } from '@libp2p/interface' +import type { Logger } from '@libp2p/logger' +import type { ProgressEvent, ProgressOptions } from 'progress-events' + +// Add a formatter for a bitswap message +debug.formatters.B = (b?: BitswapMessage): string => { + if (b == null) { + return 'undefined' + } + + return JSON.stringify({ + blocks: b.blocks?.map(b => ({ + data: `${uint8ArrayToString(b.data, 'base64').substring(0, 10)}...`, + prefix: uint8ArrayToString(b.prefix, 'base64') + })), + blockPresences: b.blockPresences?.map(p => ({ + ...p, + cid: CID.decode(p.cid).toString() + })), + wantlist: b.wantlist == null + ? undefined + : { + full: b.wantlist.full, + entries: b.wantlist.entries.map(e => ({ + ...e, + cid: CID.decode(e.cid).toString() + })) + } + }, null, 2) +} + +export type BitswapNetworkProgressEvents = + ProgressEvent<'bitswap:network:dial', PeerId> + +export type BitswapNetworkWantProgressEvents = + ProgressEvent<'bitswap:network:send-wantlist', PeerId> | + ProgressEvent<'bitswap:network:send-wantlist:error', { peer: PeerId, error: Error }> | + ProgressEvent<'bitswap:network:find-providers', CID> | + BitswapNetworkProgressEvents + +export type BitswapNetworkNotifyProgressEvents = + BitswapNetworkProgressEvents | + ProgressEvent<'bitswap:network:send-block', PeerId> + +export interface NetworkInit { + hashLoader?: MultihashHasherLoader + maxInboundStreams?: number + maxOutboundStreams?: number + messageReceiveTimeout?: number + messageSendTimeout?: number + messageSendConcurrency?: number + protocols?: string[] + runOnTransientConnections?: boolean +} + +export interface NetworkComponents { + routing: Routing + logger: ComponentLogger + libp2p: Libp2p + metrics?: Metrics +} + +export interface NetworkEvents { + 'bitswap:message': CustomEvent<{ peer: PeerId, message: BitswapMessage }> + 'peer:connected': CustomEvent + 'peer:disconnected': CustomEvent +} + +interface SendMessageJobOptions extends AbortOptions, ProgressOptions, PeerQueueJobOptions { + message: BitswapMessage +} + +export class Network extends TypedEventEmitter { + private readonly log: Logger + private readonly libp2p: Libp2p + private readonly routing: Routing + private readonly protocols: string[] + private running: boolean + private readonly maxInboundStreams: number + private readonly maxOutboundStreams: number + private readonly messageReceiveTimeout: number + private registrarIds: string[] + private readonly metrics?: { blocksSent: MetricGroup, dataSent: MetricGroup } + private readonly sendQueue: PeerQueue + private readonly messageSendTimeout: number + private readonly runOnTransientConnections: boolean + + constructor (components: NetworkComponents, init: NetworkInit = {}) { + super() + + this.log = components.logger.forComponent('helia:bitswap:network') + this.libp2p = components.libp2p + this.routing = components.routing + this.protocols = init.protocols ?? [BITSWAP_120] + this.registrarIds = [] + this.running = false + + // bind event listeners + this._onStream = this._onStream.bind(this) + this.maxInboundStreams = init.maxInboundStreams ?? DEFAULT_MAX_INBOUND_STREAMS + this.maxOutboundStreams = init.maxOutboundStreams ?? DEFAULT_MAX_OUTBOUND_STREAMS + this.messageReceiveTimeout = init.messageReceiveTimeout ?? DEFAULT_MESSAGE_RECEIVE_TIMEOUT + this.messageSendTimeout = init.messageSendTimeout ?? DEFAULT_MESSAGE_SEND_TIMEOUT + this.runOnTransientConnections = init.runOnTransientConnections ?? DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS + + if (components.metrics != null) { + this.metrics = { + blocksSent: components.metrics?.registerMetricGroup('ipfs_bitswap_sent_blocks'), + dataSent: components.metrics?.registerMetricGroup('ipfs_bitswap_sent_data_bytes') + } + } + + this.sendQueue = new PeerQueue({ + concurrency: init.messageSendConcurrency, + metrics: components.metrics, + metricName: 'ipfs_bitswap_message_send_queue' + }) + this.sendQueue.addEventListener('error', (evt) => { + this.log.error('error sending wantlist to peer', evt.detail) + }) + } + + async start (): Promise { + if (this.running) { + return + } + + this.running = true + + await this.libp2p.handle(this.protocols, this._onStream, { + maxInboundStreams: this.maxInboundStreams, + maxOutboundStreams: this.maxOutboundStreams, + runOnTransientConnection: this.runOnTransientConnections + }) + + // register protocol with topology + const topology: Topology = { + onConnect: (peerId: PeerId) => { + this.safeDispatchEvent('peer:connected', { + detail: peerId + }) + }, + onDisconnect: (peerId: PeerId) => { + this.safeDispatchEvent('peer:disconnected', { + detail: peerId + }) + } + } + + this.registrarIds = [] + + for (const protocol of this.protocols) { + this.registrarIds.push(await this.libp2p.register(protocol, topology)) + } + + // All existing connections are like new ones for us + this.libp2p.getConnections().forEach(conn => { + this.safeDispatchEvent('peer:connected', { + detail: conn.remotePeer + }) + }) + } + + async stop (): Promise { + this.running = false + + // Unhandle both, libp2p doesn't care if it's not already handled + await this.libp2p.unhandle(this.protocols) + + // unregister protocol and handlers + if (this.registrarIds != null) { + for (const id of this.registrarIds) { + this.libp2p.unregister(id) + } + + this.registrarIds = [] + } + } + + /** + * Handles incoming bitswap messages + */ + _onStream (info: IncomingStreamData): void { + if (!this.running) { + return + } + + const { stream, connection } = info + + Promise.resolve().then(async () => { + this.log('incoming new bitswap %s stream from %p', stream.protocol, connection.remotePeer) + const abortListener = (): void => { + stream.abort(new CodeError('Incoming Bitswap stream timed out', 'ERR_TIMEOUT')) + } + + let signal = AbortSignal.timeout(this.messageReceiveTimeout) + setMaxListeners(Infinity, signal) + signal.addEventListener('abort', abortListener) + + await pipe( + stream, + (source) => lp.decode(source), + async (source) => { + for await (const data of source) { + try { + const message = BitswapMessage.decode(data) + this.log('incoming new bitswap %s message from %p %B', stream.protocol, connection.remotePeer, message) + + this.safeDispatchEvent('bitswap:message', { + detail: { + peer: connection.remotePeer, + message + } + }) + + // we have received some data so reset the timeout controller + signal.removeEventListener('abort', abortListener) + signal = AbortSignal.timeout(this.messageReceiveTimeout) + setMaxListeners(Infinity, signal) + signal.addEventListener('abort', abortListener) + } catch (err: any) { + this.log.error('error reading incoming bitswap message from %p', connection.remotePeer, err) + stream.abort(err) + break + } + } + } + ) + }) + .catch(err => { + this.log.error('error handling incoming stream from %p', connection.remotePeer, err) + stream.abort(err) + }) + } + + /** + * Find providers given a `cid`. + */ + async * findProviders (cid: CID, options?: AbortOptions & ProgressOptions): AsyncIterable { + options?.onProgress?.(new CustomProgressEvent('bitswap:network:find-providers', cid)) + + for await (const provider of this.routing.findProviders(cid, options)) { + // unless we explicitly run on transient connections, skip peers that only + // have circuit relay addresses as bitswap won't run over them + if (!this.runOnTransientConnections) { + let hasDirectAddress = false + + for (let ma of provider.multiaddrs) { + if (ma.getPeerId() === null) { + ma = ma.encapsulate(`/p2p/${provider.id}`) + } + + if (!Circuit.exactMatch(ma)) { + hasDirectAddress = true + break + } + } + + if (!hasDirectAddress) { + continue + } + } + + yield provider + } + } + + /** + * Find the providers of a given `cid` and connect to them. + */ + async findAndConnect (cid: CID, options?: WantOptions): Promise { + await drain( + take( + map(this.findProviders(cid, options), async provider => this.connectTo(provider.id, options) + .catch(err => { + // Prevent unhandled promise rejection + this.log.error(err) + })), + options?.maxProviders ?? DEFAULT_MAX_PROVIDERS_PER_REQUEST + ) + ) + .catch(err => { + this.log.error(err) + }) + } + + /** + * Connect to the given peer + * Send the given msg (instance of Message) to the given peer + */ + async sendMessage (peerId: PeerId, msg: Partial, options?: AbortOptions & ProgressOptions): Promise { + if (!this.running) { + throw new Error('network isn\'t running') + } + + const message: BitswapMessage = { + wantlist: { + full: msg.wantlist?.full ?? false, + entries: msg.wantlist?.entries ?? [] + }, + blocks: msg.blocks ?? [], + blockPresences: msg.blockPresences ?? [], + pendingBytes: msg.pendingBytes ?? 0 + } + + const signal = anySignal([AbortSignal.timeout(this.messageSendTimeout), options?.signal]) + setMaxListeners(Infinity, signal) + + try { + const existingJob = this.sendQueue.find(peerId) + + if (existingJob?.status === 'queued') { + // merge messages instead of adding new job + existingJob.options.message = mergeMessages(existingJob.options.message, message) + + await existingJob.join({ + signal + }) + + return + } + + await this.sendQueue.add(async (options) => { + const message = options?.message + + if (message == null) { + throw new CodeError('No message to send', 'ERR_NO_MESSAGE') + } + + this.log('sendMessage to %p %B', peerId, message) + + options?.onProgress?.(new CustomProgressEvent('bitswap:network:send-wantlist', peerId)) + + const stream = await this.libp2p.dialProtocol(peerId, BITSWAP_120, options) + + try { + const lp = lpStream(stream) + await lp.write(BitswapMessage.encode(message), options) + await lp.unwrap().close(options) + } catch (err: any) { + options?.onProgress?.(new CustomProgressEvent<{ peer: PeerId, error: Error }>('bitswap:network:send-wantlist:error', { peer: peerId, error: err })) + this.log.error('error sending message to %p', peerId, err) + stream.abort(err) + } + + this._updateSentStats(peerId, message.blocks) + }, { + peerId, + signal, + message + }) + } finally { + signal.clear() + } + } + + /** + * Connects to another peer + */ + async connectTo (peer: PeerId, options?: AbortOptions & ProgressOptions): Promise { // eslint-disable-line require-await + if (!this.running) { + throw new CodeError('Network isn\'t running', 'ERR_NOT_STARTED') + } + + options?.onProgress?.(new CustomProgressEvent('bitswap:network:dial', peer)) + return this.libp2p.dial(peer, options) + } + + _updateSentStats (peerId: PeerId, blocks: Block[] = []): void { + if (this.metrics != null) { + let bytes = 0 + + for (const block of blocks.values()) { + bytes += block.data.byteLength + } + + this.metrics.dataSent.increment({ + global: bytes, + [peerId.toString()]: bytes + }) + this.metrics.blocksSent.increment({ + global: blocks.length, + [peerId.toString()]: blocks.length + }) + } + } +} + +function mergeMessages (messageA: BitswapMessage, messageB: BitswapMessage): BitswapMessage { + const wantListEntries = new Map( + (messageA.wantlist?.entries ?? []).map(entry => ([ + base64.encode(entry.cid), + entry + ])) + ) + + for (const entry of messageB.wantlist?.entries ?? []) { + const key = base64.encode(entry.cid) + const existingEntry = wantListEntries.get(key) + + if (existingEntry != null) { + // take highest priority + if (existingEntry.priority > entry.priority) { + entry.priority = existingEntry.priority + } + + // take later values if passed, otherwise use earlier ones + entry.cancel = entry.cancel ?? existingEntry.cancel + entry.wantType = entry.wantType ?? existingEntry.wantType + entry.sendDontHave = entry.sendDontHave ?? existingEntry.sendDontHave + } + + wantListEntries.set(key, entry) + } + + const blockPresences = new Map( + messageA.blockPresences.map(presence => ([ + base64.encode(presence.cid), + presence + ])) + ) + + for (const blockPresence of messageB.blockPresences) { + const key = base64.encode(blockPresence.cid) + + // override earlier block presence with later one as if duplicated it is + // likely to be more accurate since it is more recent + blockPresences.set(key, blockPresence) + } + + const blocks = new Map( + messageA.blocks.map(block => ([ + base64.encode(block.data), + block + ])) + ) + + for (const block of messageB.blocks) { + const key = base64.encode(block.data) + + blocks.set(key, block) + } + + const output: BitswapMessage = { + wantlist: { + full: messageA.wantlist?.full ?? messageB.wantlist?.full ?? false, + entries: [...wantListEntries.values()] + }, + blockPresences: [...blockPresences.values()], + blocks: [...blocks.values()], + pendingBytes: messageA.pendingBytes + messageB.pendingBytes + } + + return output +} diff --git a/packages/bitswap/src/notifications.ts b/packages/bitswap/src/notifications.ts new file mode 100644 index 00000000..c7075c87 --- /dev/null +++ b/packages/bitswap/src/notifications.ts @@ -0,0 +1,137 @@ +import { EventEmitter } from 'events' +import { CodeError } from '@libp2p/interface' +import { CustomProgressEvent, type ProgressOptions } from 'progress-events' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import type { BitswapWantBlockProgressEvents } from './index.js' +import type { AbortOptions, ComponentLogger, PeerId } from '@libp2p/interface' +import type { Logger } from '@libp2p/logger' +import type { CID } from 'multiformats/cid' + +/** + * Return the event name for an unwant of the passed CID + */ +export const unwantEvent = (cid: CID): string => `unwant:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` + +/** + * Return the event name for the receipt of the block for the passed CID + */ +export const receivedBlockEvent = (cid: CID): string => `block:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` + +/** + * Return the event name for a peer telling us they have the block for the + * passed CID + */ +export const haveEvent = (cid: CID): string => `have:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` + +/** + * Return the event name for a peer telling us they do not have the block for + * the passed CID + */ +export const doNotHaveEvent = (cid: CID): string => `do-not-have:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` + +export interface ReceivedBlockListener { + (block: Uint8Array, peer: PeerId): void +} + +export interface HaveBlockListener { + (peer: PeerId): void +} + +export interface DoNotHaveBlockListener { + (peer: PeerId): void +} + +export interface NotificationsComponents { + logger: ComponentLogger +} + +export class Notifications extends EventEmitter { + private readonly log: Logger + + /** + * Internal module used to track events about incoming blocks, + * wants and unwants. + */ + constructor (components: NotificationsComponents) { + super() + + this.setMaxListeners(Infinity) + this.log = components.logger.forComponent('helia:bitswap:notifications') + } + + /** + * Signal the system that the passed peer has the block + */ + haveBlock (cid: CID, peer: PeerId): void { + const event = haveEvent(cid) + this.log(event) + this.emit(event, peer) + } + + /** + * Signal the system that the passed peer does not have block + */ + doNotHaveBlock (cid: CID, peer: PeerId): void { + const event = doNotHaveEvent(cid) + this.log(event) + this.emit(event, peer) + } + + /** + * Signal the system that we received `block` from the passed peer + */ + receivedBlock (cid: CID, block: Uint8Array, peer: PeerId): void { + const event = receivedBlockEvent(cid) + this.log(event) + this.emit(event, block, peer) + } + + /** + * Signal the system that we are waiting to receive the block associated with + * the given `cid`. + * + * Returns a Promise that resolves to the block when it is received, or + * rejects if the block is unwanted. + */ + async wantBlock (cid: CID, options: AbortOptions & ProgressOptions = {}): Promise { + const blockEvt = receivedBlockEvent(cid) + const unwantEvt = unwantEvent(cid) + + this.log(`wantBlock:${cid}`) + + return new Promise((resolve, reject) => { + const onUnwant = (): void => { + this.removeListener(blockEvt, onBlock) + + options.onProgress?.(new CustomProgressEvent('bitswap:want-block:unwant', cid)) + reject(new CodeError(`Block for ${cid} unwanted`, 'ERR_UNWANTED')) + } + + const onBlock = (data: Uint8Array): void => { + this.removeListener(unwantEvt, onUnwant) + + options.onProgress?.(new CustomProgressEvent('bitswap:want-block:block', cid)) + resolve(data) + } + + this.once(unwantEvt, onUnwant) + this.once(blockEvt, onBlock) + + options.signal?.addEventListener('abort', () => { + this.removeListener(blockEvt, onBlock) + this.removeListener(unwantEvt, onUnwant) + + reject(new CodeError(`Want for ${cid} aborted`, 'ERR_ABORTED')) + }) + }) + } + + /** + * Signal that the block is not wanted any more + */ + unwantBlock (cid: CID): void { + const event = unwantEvent(cid) + this.log(event) + this.emit(event) + } +} diff --git a/packages/bitswap/src/pb/message.proto b/packages/bitswap/src/pb/message.proto new file mode 100644 index 00000000..bed8d177 --- /dev/null +++ b/packages/bitswap/src/pb/message.proto @@ -0,0 +1,42 @@ +// adapted from https://github.com/ipfs/boxo/blob/main/bitswap/message/pb/message.proto +syntax = "proto3"; + +enum WantType { + WantBlock = 0; // send me the block for the CID + WantHave = 1; // just tell me if you have the block for the CID or send it if it's really small +} + +message WantlistEntry { + bytes cid = 1; // the block cid (cidV0 in bitswap 1.0.0, cidV1 in bitswap 1.1.0) + int32 priority = 2; // the priority (normalized). default to 1 + optional bool cancel = 3; // whether this revokes an entry + optional WantType wantType = 4; // Note: defaults to enum 0, ie Block + optional bool sendDontHave = 5; // Note: defaults to false +} + +message Wantlist { + repeated WantlistEntry entries = 1; // a list of wantlist entries + optional bool full = 2; // whether this is the full wantlist. default to false +} + +message Block { + bytes prefix = 1; // CID prefix (cid version, multicodec and multihash prefix (type + length) + bytes data = 2; +} + +enum BlockPresenceType { + HaveBlock = 0; + DontHaveBlock = 1; +} + +message BlockPresence { + bytes cid = 1; + BlockPresenceType type = 2; +} + +message BitswapMessage { + Wantlist wantlist = 1; + repeated Block blocks = 3; // used to send Blocks in bitswap 1.1.0 + repeated BlockPresence blockPresences = 4; + int32 pendingBytes = 5; +} diff --git a/packages/bitswap/src/pb/message.ts b/packages/bitswap/src/pb/message.ts new file mode 100644 index 00000000..ad7b47ea --- /dev/null +++ b/packages/bitswap/src/pb/message.ts @@ -0,0 +1,450 @@ +/* eslint-disable import/export */ +/* eslint-disable complexity */ +/* eslint-disable @typescript-eslint/no-namespace */ +/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ +/* eslint-disable @typescript-eslint/no-empty-interface */ + +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' +import type { Uint8ArrayList } from 'uint8arraylist' + +export enum WantType { + WantBlock = 'WantBlock', + WantHave = 'WantHave' +} + +enum __WantTypeValues { + WantBlock = 0, + WantHave = 1 +} + +export namespace WantType { + export const codec = (): Codec => { + return enumeration(__WantTypeValues) + } +} +export interface WantlistEntry { + cid: Uint8Array + priority: number + cancel?: boolean + wantType?: WantType + sendDontHave?: boolean +} + +export namespace WantlistEntry { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.cid != null && obj.cid.byteLength > 0)) { + w.uint32(10) + w.bytes(obj.cid) + } + + if ((obj.priority != null && obj.priority !== 0)) { + w.uint32(16) + w.int32(obj.priority) + } + + if (obj.cancel != null) { + w.uint32(24) + w.bool(obj.cancel) + } + + if (obj.wantType != null) { + w.uint32(32) + WantType.codec().encode(obj.wantType, w) + } + + if (obj.sendDontHave != null) { + w.uint32(40) + w.bool(obj.sendDontHave) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + cid: uint8ArrayAlloc(0), + priority: 0 + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.cid = reader.bytes() + break + } + case 2: { + obj.priority = reader.int32() + break + } + case 3: { + obj.cancel = reader.bool() + break + } + case 4: { + obj.wantType = WantType.codec().decode(reader) + break + } + case 5: { + obj.sendDontHave = reader.bool() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, WantlistEntry.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): WantlistEntry => { + return decodeMessage(buf, WantlistEntry.codec()) + } +} + +export interface Wantlist { + entries: WantlistEntry[] + full?: boolean +} + +export namespace Wantlist { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if (obj.entries != null) { + for (const value of obj.entries) { + w.uint32(10) + WantlistEntry.codec().encode(value, w) + } + } + + if (obj.full != null) { + w.uint32(16) + w.bool(obj.full) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + entries: [] + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.entries.push(WantlistEntry.codec().decode(reader, reader.uint32())) + break + } + case 2: { + obj.full = reader.bool() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, Wantlist.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): Wantlist => { + return decodeMessage(buf, Wantlist.codec()) + } +} + +export interface Block { + prefix: Uint8Array + data: Uint8Array +} + +export namespace Block { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.prefix != null && obj.prefix.byteLength > 0)) { + w.uint32(10) + w.bytes(obj.prefix) + } + + if ((obj.data != null && obj.data.byteLength > 0)) { + w.uint32(18) + w.bytes(obj.data) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + prefix: uint8ArrayAlloc(0), + data: uint8ArrayAlloc(0) + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.prefix = reader.bytes() + break + } + case 2: { + obj.data = reader.bytes() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, Block.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): Block => { + return decodeMessage(buf, Block.codec()) + } +} + +export enum BlockPresenceType { + HaveBlock = 'HaveBlock', + DontHaveBlock = 'DontHaveBlock' +} + +enum __BlockPresenceTypeValues { + HaveBlock = 0, + DontHaveBlock = 1 +} + +export namespace BlockPresenceType { + export const codec = (): Codec => { + return enumeration(__BlockPresenceTypeValues) + } +} +export interface BlockPresence { + cid: Uint8Array + type: BlockPresenceType +} + +export namespace BlockPresence { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.cid != null && obj.cid.byteLength > 0)) { + w.uint32(10) + w.bytes(obj.cid) + } + + if (obj.type != null && __BlockPresenceTypeValues[obj.type] !== 0) { + w.uint32(16) + BlockPresenceType.codec().encode(obj.type, w) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + cid: uint8ArrayAlloc(0), + type: BlockPresenceType.HaveBlock + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.cid = reader.bytes() + break + } + case 2: { + obj.type = BlockPresenceType.codec().decode(reader) + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, BlockPresence.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): BlockPresence => { + return decodeMessage(buf, BlockPresence.codec()) + } +} + +export interface BitswapMessage { + wantlist?: Wantlist + blocks: Block[] + blockPresences: BlockPresence[] + pendingBytes: number +} + +export namespace BitswapMessage { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if (obj.wantlist != null) { + w.uint32(10) + Wantlist.codec().encode(obj.wantlist, w) + } + + if (obj.blocks != null) { + for (const value of obj.blocks) { + w.uint32(26) + Block.codec().encode(value, w) + } + } + + if (obj.blockPresences != null) { + for (const value of obj.blockPresences) { + w.uint32(34) + BlockPresence.codec().encode(value, w) + } + } + + if ((obj.pendingBytes != null && obj.pendingBytes !== 0)) { + w.uint32(40) + w.int32(obj.pendingBytes) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + blocks: [], + blockPresences: [], + pendingBytes: 0 + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.wantlist = Wantlist.codec().decode(reader, reader.uint32()) + break + } + case 3: { + obj.blocks.push(Block.codec().decode(reader, reader.uint32())) + break + } + case 4: { + obj.blockPresences.push(BlockPresence.codec().decode(reader, reader.uint32())) + break + } + case 5: { + obj.pendingBytes = reader.int32() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, BitswapMessage.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): BitswapMessage => { + return decodeMessage(buf, BitswapMessage.codec()) + } +} diff --git a/packages/bitswap/src/peer-want-lists/index.ts b/packages/bitswap/src/peer-want-lists/index.ts new file mode 100644 index 00000000..584ff5e9 --- /dev/null +++ b/packages/bitswap/src/peer-want-lists/index.ts @@ -0,0 +1,148 @@ +import { trackedPeerMap, PeerSet } from '@libp2p/peer-collections' +import { CID } from 'multiformats/cid' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { WantType } from '../pb/message.js' +import { Ledger } from './ledger.js' +import type { BitswapNotifyProgressEvents, WantListEntry } from '../index.js' +import type { Network } from '../network.js' +import type { BitswapMessage } from '../pb/message.js' +import type { ComponentLogger, Metrics, PeerId } from '@libp2p/interface' +import type { PeerMap } from '@libp2p/peer-collections' +import type { Blockstore } from 'interface-blockstore' +import type { AbortOptions } from 'it-length-prefixed-stream' +import type { ProgressOptions } from 'progress-events' + +export interface PeerWantListsInit { + maxSizeReplaceHasWithBlock?: number +} + +export interface PeerWantListsComponents { + blockstore: Blockstore + network: Network + metrics?: Metrics + logger: ComponentLogger +} + +export interface PeerLedger { + peer: PeerId + value: number + sent: number + received: number + exchanged: number +} + +export class PeerWantLists { + public blockstore: Blockstore + public network: Network + public readonly ledgerMap: PeerMap + private readonly maxSizeReplaceHasWithBlock?: number + + constructor (components: PeerWantListsComponents, init: PeerWantListsInit = {}) { + this.blockstore = components.blockstore + this.network = components.network + this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock + + this.ledgerMap = trackedPeerMap({ + name: 'ipfs_bitswap_ledger_map', + metrics: components.metrics + }) + } + + ledgerForPeer (peerId: PeerId): PeerLedger | undefined { + const ledger = this.ledgerMap.get(peerId) + + if (ledger == null) { + return undefined + } + + return { + peer: ledger.peerId, + value: ledger.debtRatio(), + sent: ledger.bytesSent, + received: ledger.bytesReceived, + exchanged: ledger.exchangeCount + } + } + + wantListForPeer (peerId: PeerId): WantListEntry[] | undefined { + const ledger = this.ledgerMap.get(peerId) + + if (ledger == null) { + return undefined + } + + return [...ledger.wants.values()] + } + + peers (): PeerId[] { + return Array.from(this.ledgerMap.values()).map((l) => l.peerId) + } + + /** + * Handle incoming messages + */ + async messageReceived (peerId: PeerId, message: BitswapMessage): Promise { + let ledger = this.ledgerMap.get(peerId) + + if (ledger == null) { + ledger = new Ledger({ + peerId, + blockstore: this.blockstore, + network: this.network + }, { + maxSizeReplaceHasWithBlock: this.maxSizeReplaceHasWithBlock + }) + this.ledgerMap.set(peerId, ledger) + } + + // record the amount of block data received + ledger.receivedBytes(message.blocks?.reduce((acc, curr) => acc + curr.data.byteLength, 0) ?? 0) + + if (message.wantlist != null) { + // if the message has a full wantlist, clear the current wantlist + if (message.wantlist.full === true) { + ledger.wants.clear() + } + + // clear cancelled wants and add new wants to the ledger + for (const entry of message.wantlist.entries) { + const cid = CID.decode(entry.cid) + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + + if (entry.cancel === true) { + ledger.wants.delete(cidStr) + } else { + ledger.wants.set(cidStr, { + cid, + session: new PeerSet(), + priority: entry.priority, + wantType: entry.wantType ?? WantType.WantBlock, + sendDontHave: entry.sendDontHave ?? false, + cancel: entry.cancel ?? false + }) + } + } + } + + await ledger.sendBlocksToPeer() + } + + async receivedBlock (cid: CID, options: ProgressOptions & AbortOptions): Promise { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + const ledgers: Ledger[] = [] + + for (const ledger of this.ledgerMap.values()) { + if (ledger.wants.has(cidStr)) { + ledgers.push(ledger) + } + } + + await Promise.all( + ledgers.map(async (ledger) => ledger.sendBlocksToPeer(options)) + ) + } + + peerDisconnected (peerId: PeerId): void { + this.ledgerMap.delete(peerId) + } +} diff --git a/packages/bitswap/src/peer-want-lists/ledger.ts b/packages/bitswap/src/peer-want-lists/ledger.ts new file mode 100644 index 00000000..314556da --- /dev/null +++ b/packages/bitswap/src/peer-want-lists/ledger.ts @@ -0,0 +1,138 @@ +/* eslint-disable max-depth */ +import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../constants.js' +import { BlockPresenceType, type BitswapMessage, WantType } from '../pb/message.js' +import { cidToPrefix } from '../utils/cid-prefix.js' +import type { WantListEntry } from '../index.js' +import type { Network } from '../network.js' +import type { PeerId } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' +import type { AbortOptions } from 'it-length-prefixed-stream' + +export interface LedgerComponents { + peerId: PeerId + blockstore: Blockstore + network: Network +} + +export interface LedgerInit { + maxSizeReplaceHasWithBlock?: number +} + +export class Ledger { + public peerId: PeerId + private readonly blockstore: Blockstore + private readonly network: Network + public wants: Map + public exchangeCount: number + public bytesSent: number + public bytesReceived: number + public lastExchange?: number + private readonly maxSizeReplaceHasWithBlock: number + + constructor (components: LedgerComponents, init: LedgerInit) { + this.peerId = components.peerId + this.blockstore = components.blockstore + this.network = components.network + this.wants = new Map() + + this.exchangeCount = 0 + this.bytesSent = 0 + this.bytesReceived = 0 + this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock ?? DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK + } + + sentBytes (n: number): void { + this.exchangeCount++ + this.lastExchange = (new Date()).getTime() + this.bytesSent += n + } + + receivedBytes (n: number): void { + this.exchangeCount++ + this.lastExchange = (new Date()).getTime() + this.bytesReceived += n + } + + debtRatio (): number { + return (this.bytesSent / (this.bytesReceived + 1)) // +1 is to prevent division by zero + } + + public async sendBlocksToPeer (options?: AbortOptions): Promise { + const message: Pick = { + blockPresences: [], + blocks: [] + } + const sentBlocks = new Set() + + for (const [key, entry] of this.wants.entries()) { + let block: Uint8Array | undefined + let has = false + + try { + block = await this.blockstore.get(entry.cid, options) + has = true + } catch (err: any) { + if (err.code !== 'ERR_NOT_FOUND') { + throw err + } + } + + if (!has) { + // we don't have the requested block and the remote is not interested + // in us telling them that + if (!entry.sendDontHave) { + continue + } + + entry.sentDontHave = true + message.blockPresences.push({ + cid: entry.cid.bytes, + type: BlockPresenceType.DontHaveBlock + }) + + continue + } + + if (block != null) { + // have the requested block + if (entry.wantType === WantType.WantHave) { + if (block.byteLength < this.maxSizeReplaceHasWithBlock) { + // send it anyway + sentBlocks.add(key) + message.blocks.push({ + data: block, + prefix: cidToPrefix(entry.cid) + }) + } else { + // tell them we have the block + message.blockPresences.push({ + cid: entry.cid.bytes, + type: BlockPresenceType.HaveBlock + }) + } + } else { + // they want the block, send it to them + sentBlocks.add(key) + message.blocks.push({ + data: block, + prefix: cidToPrefix(entry.cid) + }) + } + } + } + + // only send the message if we actually have something to send + if (message.blocks.length > 0 || message.blockPresences.length > 0) { + await this.network.sendMessage(this.peerId, message, options) + + // update accounting + this.sentBytes(message.blocks.reduce((acc, curr) => acc + curr.data.byteLength, 0)) + + // remove sent blocks from local copy of their want list - they can still + // re-request if required + for (const key of sentBlocks) { + this.wants.delete(key) + } + } + } +} diff --git a/packages/bitswap/src/session.ts b/packages/bitswap/src/session.ts new file mode 100644 index 00000000..7d0c8f2b --- /dev/null +++ b/packages/bitswap/src/session.ts @@ -0,0 +1,67 @@ +import { CodeError } from '@libp2p/interface' +import { PeerSet } from '@libp2p/peer-collections' +import type { BitswapWantProgressEvents, BitswapSession as BitswapSessionInterface } from './index.js' +import type { Network } from './network.js' +import type { Notifications } from './notifications.js' +import type { WantList } from './want-list.js' +import type { ComponentLogger, Logger } from '@libp2p/interface' +import type { AbortOptions } from 'interface-store' +import type { CID } from 'multiformats/cid' +import type { ProgressOptions } from 'progress-events' + +export interface BitswapSessionComponents { + notifications: Notifications + network: Network + wantList: WantList + logger: ComponentLogger +} + +export interface BitswapSessionInit { + root: CID +} + +class BitswapSession implements BitswapSessionInterface { + public readonly root: CID + public readonly peers: PeerSet + private readonly log: Logger + private readonly notifications: Notifications + private readonly wantList: WantList + + constructor (components: BitswapSessionComponents, init: BitswapSessionInit) { + this.peers = new PeerSet() + this.root = init.root + this.log = components.logger.forComponent(`bitswap:session:${init.root}`) + this.notifications = components.notifications + this.wantList = components.wantList + } + + async want (cid: CID, options?: AbortOptions & ProgressOptions): Promise { + if (this.peers.size === 0) { + throw new CodeError('Bitswap session had no peers', 'ERR_NO_SESSION_PEERS') + } + + // normalize to v1 CID + cid = cid.toV1() + + this.log('sending WANT-BLOCK for %c to', cid, this.peers) + + await this.wantList.wantBlocks([cid], { + session: this.peers, + sendDontHave: true + }) + + const block = await this.notifications.wantBlock(cid, options) + + this.log('sending cancels for %c to', cid, this.peers) + + await this.wantList.cancelWants([cid], { + session: this.peers + }) + + return block + } +} + +export function createBitswapSession (components: BitswapSessionComponents, init: BitswapSessionInit): BitswapSessionInterface { + return new BitswapSession(components, init) +} diff --git a/packages/bitswap/src/stats.ts b/packages/bitswap/src/stats.ts new file mode 100644 index 00000000..b95bb5a4 --- /dev/null +++ b/packages/bitswap/src/stats.ts @@ -0,0 +1,70 @@ +import { EventEmitter } from 'events' +import type { MetricGroup, Metrics, PeerId } from '@libp2p/interface' + +export interface StatsComponents { + metrics?: Metrics +} + +export class Stats extends EventEmitter { + private readonly blocksReceived?: MetricGroup + private readonly duplicateBlocksReceived?: MetricGroup + private readonly dataReceived?: MetricGroup + private readonly duplicateDataReceived?: MetricGroup + + constructor (components: StatsComponents) { + super() + + this.blocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_received_blocks') + this.duplicateBlocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_received_blocks') + this.dataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_data_received_bytes') + this.duplicateDataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_data_received_bytes') + } + + updateBlocksReceived (count: number = 1, peerId?: PeerId): void { + const stats: Record = { + global: count + } + + if (peerId != null) { + stats[peerId.toString()] = count + } + + this.blocksReceived?.increment(stats) + } + + updateDuplicateBlocksReceived (count: number = 1, peerId?: PeerId): void { + const stats: Record = { + global: count + } + + if (peerId != null) { + stats[peerId.toString()] = count + } + + this.duplicateBlocksReceived?.increment(stats) + } + + updateDataReceived (bytes: number, peerId?: PeerId): void { + const stats: Record = { + global: bytes + } + + if (peerId != null) { + stats[peerId.toString()] = bytes + } + + this.dataReceived?.increment(stats) + } + + updateDuplicateDataReceived (bytes: number, peerId?: PeerId): void { + const stats: Record = { + global: bytes + } + + if (peerId != null) { + stats[peerId.toString()] = bytes + } + + this.duplicateDataReceived?.increment(stats) + } +} diff --git a/packages/bitswap/src/utils/cid-prefix.ts b/packages/bitswap/src/utils/cid-prefix.ts new file mode 100644 index 00000000..4e17fe93 --- /dev/null +++ b/packages/bitswap/src/utils/cid-prefix.ts @@ -0,0 +1,8 @@ +import ve from './varint-encoder.js' +import type { CID } from 'multiformats/cid' + +export function cidToPrefix (cid: CID): Uint8Array { + return ve([ + cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength + ]) +} diff --git a/packages/bitswap/src/utils/varint-decoder.ts b/packages/bitswap/src/utils/varint-decoder.ts new file mode 100644 index 00000000..0c8a0609 --- /dev/null +++ b/packages/bitswap/src/utils/varint-decoder.ts @@ -0,0 +1,19 @@ +import { decode, encodingLength } from 'uint8-varint' + +function varintDecoder (buf: Uint8Array): number[] { + if (!(buf instanceof Uint8Array)) { + throw new Error('arg needs to be a Uint8Array') + } + + const result: number[] = [] + + while (buf.length > 0) { + const num = decode(buf) + result.push(num) + buf = buf.slice(encodingLength(num)) + } + + return result +} + +export default varintDecoder diff --git a/packages/bitswap/src/utils/varint-encoder.ts b/packages/bitswap/src/utils/varint-encoder.ts new file mode 100644 index 00000000..2fd7b165 --- /dev/null +++ b/packages/bitswap/src/utils/varint-encoder.ts @@ -0,0 +1,18 @@ +import { encode, encodingLength } from 'uint8-varint' + +function varintEncoder (buf: number[]): Uint8Array { + let out = new Uint8Array(buf.reduce((acc, curr) => { + return acc + encodingLength(curr) + }, 0)) + let offset = 0 + + for (const num of buf) { + out = encode(num, out, offset) + + offset += encodingLength(num) + } + + return out +} + +export default varintEncoder diff --git a/packages/bitswap/src/want-list.ts b/packages/bitswap/src/want-list.ts new file mode 100644 index 00000000..88b4d703 --- /dev/null +++ b/packages/bitswap/src/want-list.ts @@ -0,0 +1,271 @@ +import { trackedPeerMap, PeerSet } from '@libp2p/peer-collections' +import { trackedMap } from '@libp2p/utils/tracked-map' +import all from 'it-all' +import filter from 'it-filter' +import map from 'it-map' +import { pipe } from 'it-pipe' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { WantType } from './pb/message.js' +import type { WantListEntry } from './index.js' +import type { Network } from './network.js' +import type { BitswapMessage } from './pb/message.js' +import type { ComponentLogger, Metrics, PeerId, Startable } from '@libp2p/interface' +import type { Logger } from '@libp2p/logger' +import type { PeerMap } from '@libp2p/peer-collections' +import type { CID } from 'multiformats/cid' + +export interface WantListComponents { + network: Network + logger: ComponentLogger + metrics?: Metrics +} + +export interface WantBlocksOptions { + /** + * If set, this wantlist entry will only be sent to peers in the peer set + */ + session?: PeerSet + + /** + * Allow prioritsing blocks + */ + priority?: number + + /** + * Specify if the remote should send us the block or just tell us they have + * the block + */ + wantType?: WantType + + /** + * Pass true to get the remote to tell us if they don't have the block rather + * than not replying at all + */ + sendDontHave?: boolean + + /** + * Pass true to cancel wants with peers + */ + cancel?: boolean +} + +export class WantList implements Startable { + /** + * Tracks what CIDs we've previously sent to which peers + */ + public readonly peers: PeerMap> + public readonly wants: Map + private readonly network: Network + private readonly log: Logger + + constructor (components: WantListComponents) { + this.peers = trackedPeerMap({ + name: 'ipfs_bitswap_peers', + metrics: components.metrics + }) + this.wants = trackedMap({ + name: 'ipfs_bitswap_wantlist', + metrics: components.metrics + }) + this.network = components.network + this.log = components.logger.forComponent('helia:bitswap:wantlist:self') + } + + async _addEntries (cids: CID[], options: WantBlocksOptions = {}): Promise { + for (const cid of cids) { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + let entry = this.wants.get(cidStr) + + if (entry == null) { + // we are cancelling a want that's not in our wantlist + if (options.cancel === true) { + continue + } + + entry = { + cid, + session: options.session ?? new PeerSet(), + priority: options.priority ?? 1, + wantType: options.wantType ?? WantType.WantBlock, + cancel: Boolean(options.cancel), + sendDontHave: Boolean(options.sendDontHave) + } + } + + // upgrade want-have to want-block + if (entry.wantType === WantType.WantHave && options.wantType === WantType.WantBlock) { + entry.wantType = WantType.WantBlock + } + + // cancel the want if requested to do so + if (options.cancel === true) { + entry.cancel = true + } + + // if this entry has previously been part of a session but the new want + // is not, make this want a non-session want + if (options.session == null) { + entry.session = new PeerSet() + } + + this.wants.set(cidStr, entry) + } + + // broadcast changes + await this.sendMessages() + } + + async sendMessages (): Promise { + for (const [peerId, sentWants] of this.peers) { + const sent = new Set() + const message: Partial = { + wantlist: { + full: false, + entries: pipe( + this.wants.entries(), + (source) => filter(source, ([key, entry]) => { + // skip session-only wants + if (entry.session.size > 0 && !entry.session.has(peerId)) { + return false + } + + const sentPreviously = sentWants.has(key) + + // don't cancel if we've not sent it to them before + if (entry.cancel) { + return sentPreviously + } + + // only send if we've not sent it to them before + return !sentPreviously + }), + (source) => map(source, ([key, entry]) => { + sent.add(key) + + return { + cid: entry.cid.bytes, + priority: entry.priority, + wantType: entry.wantType, + cancel: entry.cancel, + sendDontHave: entry.sendDontHave + } + }), + (source) => all(source) + ) + } + } + + if (message.wantlist?.entries.length === 0) { + return + } + + // add message to send queue + try { + await this.network.sendMessage(peerId, message) + + // update list of messages sent to remote + for (const key of sent) { + sentWants.add(key) + } + } catch (err: any) { + this.log.error('error sending full wantlist to new peer', err) + } + } + + // queued all message sends, remove cancelled wants from wantlist and sent + // wants + for (const [key, entry] of this.wants) { + if (entry.cancel) { + this.wants.delete(key) + + for (const sentWants of this.peers.values()) { + sentWants.delete(key) + } + } + } + } + + /** + * Add all the CIDs to the wantlist + */ + async wantBlocks (cids: CID[], options: WantBlocksOptions = {}): Promise { + await this._addEntries(cids, options) + } + + /** + * Remove CIDs from the wantlist without sending cancel messages + */ + unwantBlocks (cids: CID[], options: WantBlocksOptions = {}): void { + cids.forEach(cid => { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + + this.wants.delete(cidStr) + }) + } + + /** + * Send cancel messages to peers for the passed CIDs + */ + async cancelWants (cids: CID[], options: WantBlocksOptions = {}): Promise { + this.log('cancel wants: %s', cids.length) + await this._addEntries(cids, { + ...options, + cancel: true + }) + } + + async connected (peerId: PeerId): Promise { + const sentWants = new Set() + + // new peer, give them the full wantlist + const message: Partial = { + wantlist: { + full: true, + entries: pipe( + this.wants.entries(), + (source) => filter(source, ([key, entry]) => !entry.cancel && (entry.session.size > 0 && !entry.session.has(peerId))), + (source) => filter(source, ([key, entry]) => !entry.cancel), + (source) => map(source, ([key, entry]) => { + sentWants.add(key) + + return { + cid: entry.cid.bytes, + priority: 1, + wantType: WantType.WantBlock, + cancel: false, + sendDontHave: false + } + }), + (source) => all(source) + ) + } + } + + // only send the wantlist if we have something to send + if (message.wantlist?.entries.length === 0) { + this.peers.set(peerId, sentWants) + + return + } + + try { + await this.network.sendMessage(peerId, message) + + this.peers.set(peerId, sentWants) + } catch (err) { + this.log.error('error sending full wantlist to new peer %p', peerId, err) + } + } + + disconnected (peerId: PeerId): void { + this.peers.delete(peerId) + } + + start (): void { + + } + + stop (): void { + this.peers.clear() + } +} diff --git a/packages/bitswap/test/bitswap.spec.ts b/packages/bitswap/test/bitswap.spec.ts new file mode 100644 index 00000000..650cb71a --- /dev/null +++ b/packages/bitswap/test/bitswap.spec.ts @@ -0,0 +1,451 @@ +import { start, stop } from '@libp2p/interface' +import { matchPeerId } from '@libp2p/interface-compliance-tests/matchers' +import { mockStream } from '@libp2p/interface-compliance-tests/mocks' +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { multiaddr } from '@multiformats/multiaddr' +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import delay from 'delay' +import { duplexPair } from 'it-pair/duplex' +import { pbStream } from 'it-protobuf-stream' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' +import Sinon from 'sinon' +import { stubInterface } from 'sinon-ts' +import { Bitswap } from '../src/bitswap.js' +import { DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY } from '../src/constants.js' +import { BitswapMessage, BlockPresenceType } from '../src/pb/message.js' +import { cidToPrefix } from '../src/utils/cid-prefix.js' +import type { Routing } from '@helia/interface' +import type { Connection, Libp2p, PeerId } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' +import type { StubbedInstance } from 'sinon-ts' + +interface StubbedBitswapComponents { + peerId: PeerId + routing: StubbedInstance + blockstore: Blockstore + libp2p: StubbedInstance +} + +describe('bitswap', () => { + let components: StubbedBitswapComponents + let bitswap: Bitswap + let cid: CID + let block: Uint8Array + + beforeEach(async () => { + block = Uint8Array.from([0, 1, 2, 3, 4]) + const mh = await sha256.digest(block) + cid = CID.createV0(mh).toV1() + + components = { + peerId: await createEd25519PeerId(), + routing: stubInterface(), + blockstore: new MemoryBlockstore(), + libp2p: stubInterface() + } + + bitswap = new Bitswap({ + ...components, + logger: defaultLogger() + }) + + components.libp2p.getConnections.returns([]) + + await start(bitswap) + }) + + afterEach(async () => { + if (bitswap != null) { + await stop(bitswap) + } + }) + + describe('session', () => { + it('should create a session', async () => { + const connectedPeer = await createEd25519PeerId() + + // notify topology of connected peer that supports bitswap + components.libp2p.register.getCall(0).args[1]?.onConnect?.(connectedPeer, stubInterface({ + remotePeer: connectedPeer + })) + + // the current peer does not have the block + stubPeerResponse(components.libp2p, connectedPeer, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + // providers found via routing + const providers = [{ + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/41.41.41.41/tcp/1234') + ], + protocols: ['transport-bitswap'] + }, { + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/42.42.42.42/tcp/1234') + ], + protocols: ['transport-bitswap'] + }, { + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/43.43.43.43/tcp/1234') + ], + protocols: ['transport-bitswap'] + }, { + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/44.44.44.44/tcp/1234') + ], + protocols: ['transport-bitswap'] + }, { + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/45.45.45.45/tcp/1234') + ], + protocols: ['transport-bitswap'] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + // stub first three provider responses, all but #3 have the block, second + // provider sends the block in the response + stubPeerResponse(components.libp2p, providers[0].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.HaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[1].id, { + blockPresences: [], + blocks: [{ + prefix: cidToPrefix(cid), + data: block + }], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[2].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[3].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.HaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[4].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.HaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + const session = await bitswap.createSession(cid) + expect(session.peers.size).to.equal(DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY) + expect([...session.peers].map(p => p.toString())).to.include(providers[0].id.toString()) + + // dialed connected peer first + expect(connectedPeer.equals(components.libp2p.dialProtocol.getCall(0).args[0].toString())).to.be.true() + + // dialed first provider second + expect(providers[0].id.equals(components.libp2p.dialProtocol.getCall(1).args[0].toString())).to.be.true() + + // the query continues after the session is ready + await delay(100) + + // should have continued querying until we reach DEFAULT_MAX_PROVIDERS_PER_REQUEST + expect(providers[1].id.equals(components.libp2p.dialProtocol.getCall(2).args[0].toString())).to.be.true() + expect(providers[2].id.equals(components.libp2p.dialProtocol.getCall(3).args[0].toString())).to.be.true() + + // one current peer and providers 1-4 + expect(components.libp2p.dialProtocol.callCount).to.equal(5) + + // should have stopped at DEFAULT_MAX_PROVIDERS_PER_REQUEST + expect(session.peers.size).to.equal(DEFAULT_MAX_PROVIDERS_PER_REQUEST) + }) + + it('should error when creating a session when no peers or providers have the block', async () => { + const connectedPeer = await createEd25519PeerId() + + // notify topology of connected peer that supports bitswap + components.libp2p.register.getCall(0).args[1]?.onConnect?.(connectedPeer, stubInterface({ + remotePeer: connectedPeer + })) + + // the current peer does not have the block + stubPeerResponse(components.libp2p, connectedPeer, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + // providers found via routing + const providers = [{ + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/41.41.41.41/tcp/1234') + ], + protocols: ['transport-bitswap'] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + // the provider doesn't have the block + stubPeerResponse(components.libp2p, providers[0].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + await expect(bitswap.createSession(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_PROVIDERS_FOUND') + }) + + it('should error when creating a session when no providers have the block', async () => { + // providers found via routing + const providers = [{ + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/41.41.41.41/tcp/1234') + ], + protocols: ['transport-bitswap'] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + // the provider doesn't have the block + stubPeerResponse(components.libp2p, providers[0].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + await expect(bitswap.createSession(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_PROVIDERS_FOUND') + }) + + it('should error when creating a session when no peers have the block', async () => { + const connectedPeer = await createEd25519PeerId() + + // notify topology of connected peer that supports bitswap + components.libp2p.register.getCall(0).args[1]?.onConnect?.(connectedPeer, stubInterface({ + remotePeer: connectedPeer + })) + + // the current peer does not have the block + stubPeerResponse(components.libp2p, connectedPeer, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + components.routing.findProviders.withArgs(cid).returns((async function * () {})()) + + await expect(bitswap.createSession(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_PROVIDERS_FOUND') + }) + + it('should error when creating a session when there are peers and no providers found', async () => { + components.routing.findProviders.withArgs(cid).returns((async function * () {})()) + + await expect(bitswap.createSession(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_PROVIDERS_FOUND') + }) + }) + + describe('want', () => { + it('should want a block that is in the blockstore', async () => { + await components.blockstore.put(cid, block) + + const b = await bitswap.want(cid) + + expect(b).to.equalBytes(block) + }) + + it('should want a block that is available on the network', async () => { + const remotePeer = await createEd25519PeerId() + const wantBlockSpy = Sinon.spy(bitswap.notifications, 'wantBlock') + const addToWantlistSpy = Sinon.spy(bitswap.wantList, 'wantBlocks') + const findProvsSpy = Sinon.spy(bitswap.network, 'findAndConnect') + + const p = bitswap.want(cid) + + // provider sends message + await bitswap._receiveMessage(remotePeer, { + blocks: [{ + prefix: cidToPrefix(cid), + data: block + }], + blockPresences: [], + pendingBytes: 0 + }) + + const b = await p + + // should have added cid to wantlist and searched for providers + expect(addToWantlistSpy.called).to.be.true() + expect(findProvsSpy.called).to.be.true() + + // should have cancelled the notification request + expect(wantBlockSpy.called).to.be.true() + expect(wantBlockSpy.getCall(0)).to.have.nested.property('args[1].signal.aborted', true) + + expect(b).to.equalBytes(block) + }) + + it('should abort wanting a block that is not available on the network', async () => { + const wantBlockSpy = Sinon.spy(bitswap.notifications, 'wantBlock') + + const p = bitswap.want(cid, { + signal: AbortSignal.timeout(100) + }) + + await expect(p).to.eventually.be.rejected + .with.property('code', 'ERR_ABORTED') + + // should have cancelled the notification request + expect(wantBlockSpy.called).to.be.true() + expect(wantBlockSpy.getCall(0)).to.have.nested.property('args[1].signal.aborted', true) + }) + + it('should notify peers we have a block', async () => { + const wantEventListener = bitswap.notifications.wantBlock(cid) + const receivedBlockSpy = Sinon.spy(bitswap.peerWantLists, 'receivedBlock') + + await bitswap.notify(cid, block) + + await expect(wantEventListener).to.eventually.deep.equal(block) + expect(receivedBlockSpy.called).to.be.true() + }) + }) + + describe('wantlist', () => { + it('should remove CIDs from the wantlist when the block arrives', async () => { + const remotePeer = await createEd25519PeerId() + expect(bitswap.getWantlist()).to.be.empty() + + const p = bitswap.want(cid) + + expect(bitswap.getWantlist().map(w => w.cid)).to.include(cid) + + // provider sends message + await bitswap._receiveMessage(remotePeer, { + blocks: [{ + prefix: cidToPrefix(cid), + data: block + }], + blockPresences: [], + pendingBytes: 0 + }) + + const b = await p + + expect(bitswap.getWantlist()).to.be.empty() + expect(b).to.equalBytes(block) + }) + + it('should remove CIDs from the wantlist when the want is cancelled', async () => { + expect(bitswap.getWantlist()).to.be.empty() + + const p = bitswap.want(cid, { + signal: AbortSignal.timeout(100) + }) + + expect(bitswap.getWantlist().map(w => w.cid)).to.include(cid) + + await expect(p).to.eventually.be.rejected + .with.property('code', 'ERR_ABORTED') + + expect(bitswap.getWantlist()).to.be.empty() + }) + }) + + describe('peer wantlist', () => { + it('should return a peer wantlist', async () => { + const remotePeer = await createEd25519PeerId() + + // don't have this peer yet + expect(bitswap.getPeerWantlist(remotePeer)).to.be.undefined() + + await bitswap.peerWantLists.messageReceived(remotePeer, { + wantlist: { + full: false, + entries: [{ + cid: cid.bytes, + priority: 100 + }] + }, + blockPresences: [], + blocks: [], + pendingBytes: 0 + }) + + expect(bitswap.getPeerWantlist(remotePeer)?.map(entry => entry.cid)).to.deep.equal([cid]) + }) + }) +}) + +function stubPeerResponse (libp2p: StubbedInstance, peerId: PeerId, response: BitswapMessage): void { + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + + libp2p.dialProtocol.withArgs(matchPeerId(peerId)).resolves(remoteStream) + + const connection = stubInterface({ + remotePeer: peerId + }) + + const pbstr = pbStream(localStream).pb(BitswapMessage) + void pbstr.read().then(async message => { + // after reading message from remote, open a new stream on the remote and + // send the response + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + + const onStream = libp2p.handle.getCall(0).args[1] + onStream({ stream: remoteStream, connection }) + + const pbstr = pbStream(localStream).pb(BitswapMessage) + await pbstr.write(response) + }) +} diff --git a/packages/bitswap/test/network.spec.ts b/packages/bitswap/test/network.spec.ts new file mode 100644 index 00000000..0e8cdacc --- /dev/null +++ b/packages/bitswap/test/network.spec.ts @@ -0,0 +1,445 @@ +import { mockStream } from '@libp2p/interface-compliance-tests/mocks' +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { multiaddr } from '@multiformats/multiaddr' +import { expect } from 'aegir/chai' +import delay from 'delay' +import all from 'it-all' +import { lpStream } from 'it-length-prefixed-stream' +import { duplexPair } from 'it-pair/duplex' +import { pbStream } from 'it-protobuf-stream' +import { CID } from 'multiformats/cid' +import { pEvent } from 'p-event' +import pRetry from 'p-retry' +import Sinon from 'sinon' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { BITSWAP_120 } from '../src/constants.js' +import { Network } from '../src/network.js' +import { BitswapMessage, BlockPresenceType } from '../src/pb/message.js' +import { cidToPrefix } from '../src/utils/cid-prefix.js' +import type { Routing } from '@helia/interface' +import type { Connection, Libp2p, PeerId } from '@libp2p/interface' + +interface StubbedNetworkComponents { + routing: StubbedInstance + libp2p: StubbedInstance +} + +describe('network', () => { + let network: Network + let components: StubbedNetworkComponents + + beforeEach(async () => { + components = { + routing: stubInterface(), + libp2p: stubInterface({ + getConnections: () => [] + }) + } + + network = new Network({ + ...components, + logger: defaultLogger() + }, { + messageReceiveTimeout: 100 + }) + + await network.start() + }) + + afterEach(async () => { + if (network != null) { + await network.stop() + } + }) + + it('should not connect if not running', async () => { + await network.stop() + + const peerId = await createEd25519PeerId() + + await expect(network.connectTo(peerId)) + .to.eventually.be.rejected.with.property('code', 'ERR_NOT_STARTED') + }) + + it('should register protocol handlers', () => { + expect(components.libp2p.handle.called).to.be.true() + expect(components.libp2p.register.calledWith(BITSWAP_120)).to.be.true() + }) + + it('should deregister protocol handlers', async () => { + await network.stop() + + expect(components.libp2p.unhandle.called).to.be.true() + }) + + it('should start twice', async () => { + expect(components.libp2p.handle.calledOnce).to.be.true() + + await network.start() + + expect(components.libp2p.handle.calledOnce).to.be.true() + }) + + it('should emit a bitswap:message event when receiving an incoming message', async () => { + const remotePeer = await createEd25519PeerId() + const connection = stubInterface({ + remotePeer + }) + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + const handler = components.libp2p.handle.getCall(0).args[1] + + const messageEventPromise = pEvent<'bitswap:message', CustomEvent<{ peer: PeerId, message: BitswapMessage }>>(network, 'bitswap:message') + + handler({ + stream: remoteStream, + connection + }) + + const pbstr = pbStream(localStream).pb(BitswapMessage) + await pbstr.write({ + blockPresences: [], + blocks: [], + wantlist: { + full: true, + entries: [] + }, + pendingBytes: 0 + }) + + const event = await messageEventPromise + + expect(event.detail.peer.toString()).to.equal(remotePeer.toString()) + expect(event.detail).to.have.nested.property('message.wantlist.full', true) + }) + + it('should close the stream if parsing an incoming message fails', async () => { + const remotePeer = await createEd25519PeerId() + const connection = stubInterface({ + remotePeer + }) + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + const handler = components.libp2p.handle.getCall(0).args[1] + + const spy = Sinon.spy(remoteStream, 'abort') + + handler({ + stream: remoteStream, + connection + }) + + const lpstr = lpStream(localStream) + + // garbage data, cannot be unmarshalled as protobuf + await lpstr.write(Uint8Array.from([0, 1, 2, 3])) + + await pRetry(() => { + expect(spy.called).to.be.true() + }) + }) + + it('should close the stream if no message is received', async () => { + const remotePeer = await createEd25519PeerId() + const connection = stubInterface({ + remotePeer + }) + const [, remoteDuplex] = duplexPair() + const remoteStream = mockStream(remoteDuplex) + const handler = components.libp2p.handle.getCall(0).args[1] + + const spy = Sinon.spy(remoteStream, 'abort') + + handler({ + stream: remoteStream, + connection + }) + + await pRetry(() => { + expect(spy.called).to.be.true() + }) + }) + + it('should find providers', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + const output = await all(network.findProviders(cid)) + + expect(output).to.have.lengthOf(1) + expect(output[0].id.toString()).to.equal(peerId.toString()) + expect(output[0].multiaddrs).to.have.lengthOf(1) + expect(output[0].multiaddrs[0].toString()).to.equal('/ip4/127.0.0.1/tcp/4001') + }) + + it('should ignore providers with only transient addresses', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001/p2p-circuit') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + const output = await all(network.findProviders(cid)) + + expect(output).to.be.empty() + }) + + it('should find providers with only transient addresses when running on transient connections', async () => { + await network.stop() + network = new Network({ + ...components, + logger: defaultLogger() + }, { + runOnTransientConnections: true + }) + + await network.start() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001/p2p-circuit') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + const output = await all(network.findProviders(cid)) + + expect(output).to.have.lengthOf(1) + expect(output[0].id.toString()).to.equal(peerId.toString()) + expect(output[0].multiaddrs).to.have.lengthOf(1) + expect(output[0].multiaddrs[0].toString()).to.equal('/ip4/127.0.0.1/tcp/4001/p2p-circuit') + }) + + it('should find and connect to a peer', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + await network.findAndConnect(cid) + + expect(components.libp2p.dial.calledWith(peerId)).to.be.true() + }) + + it('should find and connect to a peer with only a transient address when running on transient connections', async () => { + await network.stop() + network = new Network({ + ...components, + logger: defaultLogger() + }, { + runOnTransientConnections: true + }) + await network.start() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001/p2p-circuit') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + await network.findAndConnect(cid) + + expect(components.libp2p.dial.calledWith(peerId)).to.be.true() + }) + + it('should send a message', async () => { + const peerId = await createEd25519PeerId() + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + + components.libp2p.dialProtocol.withArgs(peerId, BITSWAP_120).resolves(remoteStream) + + void network.sendMessage(peerId, { + blocks: [], + blockPresences: [], + wantlist: { + full: true, + entries: [] + }, + pendingBytes: 0 + }) + + const pbstr = pbStream(localStream).pb(BitswapMessage) + const message = await pbstr.read() + + expect(message).to.have.nested.property('wantlist.full').that.is.true() + }) + + it('should merge messages sent to the same peer', async () => { + await network.stop() + network = new Network({ + ...components, + logger: defaultLogger() + }, { + messageSendConcurrency: 1 + }) + await network.start() + + const peerId = await createEd25519PeerId() + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + + components.libp2p.dialProtocol.withArgs(peerId, BITSWAP_120).resolves(remoteStream) + + const cid1 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3A') + const cid2 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3B') + const cid3 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3C') + const cid4 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3D') + const cid5 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3E') + const cid6 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const cid7 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3G') + + const messageA = { + blocks: [{ + prefix: cidToPrefix(cid1), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }], + blockPresences: [{ + cid: cid3.bytes, + type: BlockPresenceType.DontHaveBlock + }, { + cid: cid5.bytes, + type: BlockPresenceType.DontHaveBlock + }], + wantlist: { + full: true, + entries: [{ + cid: cid5.bytes, + priority: 5 + }, { + cid: cid6.bytes, + priority: 100 + }] + }, + pendingBytes: 5 + } + + const messageB = { + blocks: [{ + prefix: cidToPrefix(cid2), + data: Uint8Array.from([5, 6, 7, 8]) + }], + blockPresences: [{ + cid: cid4.bytes, + type: BlockPresenceType.DontHaveBlock + }, { + cid: cid5.bytes, + type: BlockPresenceType.HaveBlock + }], + wantlist: { + full: false, + entries: [{ + cid: cid6.bytes, + priority: 0 + }, { + cid: cid7.bytes, + priority: 0 + }] + }, + pendingBytes: 7 + } + + // block the queue with a slow request + const slowPeer = await createEd25519PeerId() + components.libp2p.dialProtocol.withArgs(slowPeer).callsFake(async () => { + await delay(100) + throw new Error('Urk!') + }) + void network.sendMessage(slowPeer, { + blocks: [{ + prefix: cidToPrefix(cid1), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }] + }) + + // send two messages while the queue is blocked + void network.sendMessage(peerId, messageA) + void network.sendMessage(peerId, messageB) + + // wait for long enough that we are sure we don't dial peerId twice + await delay(500) + + // one dial for slowPeer, one for peerId + expect(components.libp2p.dialProtocol).to.have.property('callCount', 2, 'made too many dials') + + const pbstr = pbStream(localStream).pb(BitswapMessage) + const message = await pbstr.read() + + expect(message).to.have.deep.property('blocks', [ + ...messageA.blocks, + ...messageB.blocks + ]) + expect(message).to.have.deep.property('blockPresences', [{ + cid: cid3.bytes, + type: BlockPresenceType.DontHaveBlock + }, { + cid: cid5.bytes, + type: BlockPresenceType.HaveBlock + }, { + cid: cid4.bytes, + type: BlockPresenceType.DontHaveBlock + }]) + expect(message).to.have.deep.property('wantlist', { + full: true, + entries: [{ + cid: cid5.bytes, + priority: 5 + }, { + cid: cid6.bytes, + priority: 100 + }, { + cid: cid7.bytes, + priority: 0 + }] + }) + expect(message).to.have.property('pendingBytes', messageA.pendingBytes + messageB.pendingBytes) + }) +}) diff --git a/packages/bitswap/test/notifications.spec.ts b/packages/bitswap/test/notifications.spec.ts new file mode 100644 index 00000000..df1d6166 --- /dev/null +++ b/packages/bitswap/test/notifications.spec.ts @@ -0,0 +1,71 @@ +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { pEvent } from 'p-event' +import { Notifications, doNotHaveEvent, haveEvent } from '../src/notifications.js' + +describe('notifications', () => { + let notifications: Notifications + + before(() => { + notifications = new Notifications({ + logger: defaultLogger() + }) + }) + + it('should notify wants after receiving a block', async () => { + const peerId = await createEd25519PeerId() + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const data = Uint8Array.from([0, 1, 2, 3, 4]) + + const p = notifications.wantBlock(cid) + + notifications.receivedBlock(cid, data, peerId) + + const block = await p + + expect(block).to.equalBytes(data) + }) + + it('should notify wants after unwanting a block', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const p = notifications.wantBlock(cid) + + notifications.unwantBlock(cid) + + await expect(p).to.eventually.rejected.with.property('code', 'ERR_UNWANTED') + }) + + it('should notify wants aborting wanting a block', async () => { + const signal = AbortSignal.timeout(100) + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const p = notifications.wantBlock(cid, { + signal + }) + + await expect(p).to.eventually.rejected.with.property('code', 'ERR_ABORTED') + }) + + it('should notify on have', async () => { + const peerId = await createEd25519PeerId() + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + const p = pEvent(notifications, haveEvent(cid)) + + notifications.haveBlock(cid, peerId) + + await expect(p).to.eventually.equal(peerId) + }) + + it('should notify on do not have', async () => { + const peerId = await createEd25519PeerId() + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + const p = pEvent(notifications, doNotHaveEvent(cid)) + + notifications.doNotHaveBlock(cid, peerId) + + await expect(p).to.eventually.equal(peerId) + }) +}) diff --git a/packages/bitswap/test/peer-want-list.spec.ts b/packages/bitswap/test/peer-want-list.spec.ts new file mode 100644 index 00000000..36dddeb8 --- /dev/null +++ b/packages/bitswap/test/peer-want-list.spec.ts @@ -0,0 +1,501 @@ +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import delay from 'delay' +import { CID } from 'multiformats/cid' +import pRetry from 'p-retry' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../src/constants.js' +import { BlockPresenceType, WantType } from '../src/pb/message.js' +import { PeerWantLists } from '../src/peer-want-lists/index.js' +import ve from '../src/utils/varint-encoder.js' +import type { Network } from '../src/network.js' +import type { ComponentLogger, PeerId } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' + +interface PeerWantListsComponentStubs { + peerId: PeerId + blockstore: Blockstore + network: StubbedInstance + logger: ComponentLogger +} + +describe('peer-want-lists', () => { + let components: PeerWantListsComponentStubs + let wantLists: PeerWantLists + + beforeEach(async () => { + components = { + peerId: await createEd25519PeerId(), + blockstore: new MemoryBlockstore(), + network: stubInterface(), + logger: defaultLogger() + } + + wantLists = new PeerWantLists(components) + }) + + it('should keep a ledger for a peer', async () => { + const remotePeer = await createEd25519PeerId() + + expect(wantLists.ledgerForPeer(remotePeer)).to.be.undefined('should not have list initially') + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + full: true, + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + }) + + const ledger = wantLists.ledgerForPeer(remotePeer) + + expect(ledger).to.have.property('peer', remotePeer) + expect(ledger).to.have.property('value', 0) + expect(ledger).to.have.property('sent', 0) + expect(ledger).to.have.property('received', 0) + expect(ledger).to.have.property('exchanged', 1) + }) + + it('should replace the wantlist for a peer when the full list is received', async () => { + const remotePeer = await createEd25519PeerId() + + const cid1 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const cid2 = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + + // first wantlist + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid1.bytes, + priority: 1 + }] + } + }) + + let entries = wantLists.wantListForPeer(remotePeer) + + expect(entries?.map(entry => entry.cid.toString())).to.include(cid1.toString()) + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + full: true, + entries: [{ + cid: cid2.bytes, + priority: 1 + }] + } + }) + + entries = wantLists.wantListForPeer(remotePeer) + + // should only have CIDs from the second message + expect(entries?.map(entry => entry.cid.toString())).to.not.include(cid1.toString()) + expect(entries?.map(entry => entry.cid.toString())).to.include(cid2.toString()) + }) + + it('should merge the wantlist for a peer when a partial list is received', async () => { + const remotePeer = await createEd25519PeerId() + + const cid1 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const cid2 = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + + // first wantlist + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid1.bytes, + priority: 1 + }] + } + }) + + let entries = wantLists.wantListForPeer(remotePeer) + + expect(entries?.map(entry => entry.cid.toString())).to.include(cid1.toString()) + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid2.bytes, + priority: 1 + }] + } + }) + + entries = wantLists.wantListForPeer(remotePeer) + + // should have both CIDs + expect(entries?.map(entry => entry.cid.toString())).to.include(cid1.toString()) + expect(entries?.map(entry => entry.cid.toString())).to.include(cid2.toString()) + }) + + it('should record the amount of incoming data', async () => { + const remotePeer = await createEd25519PeerId() + + await wantLists.messageReceived(remotePeer, { + blocks: [{ + prefix: Uint8Array.from([0, 1, 2, 3, 4]), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + prefix: Uint8Array.from([0, 1, 2]), + data: Uint8Array.from([0, 1, 2]) + }], + blockPresences: [], + pendingBytes: 0 + }) + + const ledger = wantLists.ledgerForPeer(remotePeer) + + expect(ledger).to.have.property('received', 8) + }) + + it('should send requested blocks to peer', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3, 4]) + + // we have block + await components.blockstore.put(cid, block) + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + }) + + // wait for network send + await pRetry(() => { + if (!components.network.sendMessage.called) { + throw new Error('Network message not sent') + } + }) + + const message = components.network.sendMessage.getCall(0).args[1] + + expect(message.blocks).to.have.lengthOf(1) + expect(message.blocks?.[0].data).to.equalBytes(block) + expect(message.blocks?.[0].prefix).to.equalBytes(ve([ + cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength + ])) + + // have to wait for network send + await delay(1) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.not.include(cid.toString()) + }) + + it('should send requested block presences to peer', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from(new Array(DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK + 1)) + + // we have block + await components.blockstore.put(cid, block) + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave + }] + } + }) + + // wait for network send + await pRetry(() => { + if (!components.network.sendMessage.called) { + throw new Error('Network message not sent') + } + }) + + const message = components.network.sendMessage.getCall(0).args[1] + + expect(message.blocks).to.be.empty('should not have sent blocks') + expect(message.blockPresences).to.have.lengthOf(1) + expect(message.blockPresences?.[0].cid).to.equalBytes(cid.bytes) + expect(message.blockPresences?.[0].type).to.equal(BlockPresenceType.HaveBlock, 'should have sent HaveBlock presence') + }) + + it('should send requested lack of block presences to peer', async () => { + const remotePeer = await createEd25519PeerId() + + // CID for a block we don't have + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantBlock, + sendDontHave: true + }] + } + }) + + // wait for network send + await pRetry(() => { + if (!components.network.sendMessage.called) { + throw new Error('Network message not sent') + } + }) + + const message = components.network.sendMessage.getCall(0).args[1] + + expect(message.blocks).to.be.empty('should not have sent blocks') + expect(message.blockPresences).to.have.lengthOf(1) + expect(message.blockPresences?.[0].cid).to.equalBytes(cid.bytes) + expect(message.blockPresences?.[0].type).to.equal(BlockPresenceType.DontHaveBlock, 'should have sent DontHaveBlock presence') + }) + + it('should send requested blocks to peer when presence was requested but block size is less than maxSizeReplaceHasWithBlock', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3, 4]) + + // we have block + await components.blockstore.put(cid, block) + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave + }] + } + }) + + // wait for network send + await pRetry(() => { + if (!components.network.sendMessage.called) { + throw new Error('Network message not sent') + } + }) + + const message = components.network.sendMessage.getCall(0).args[1] + + expect(message.blockPresences).to.be.empty() + expect(message.blocks).to.have.lengthOf(1) + expect(message.blocks?.[0].data).to.equalBytes(block) + expect(message.blocks?.[0].prefix).to.equalBytes(ve([ + cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength + ])) + + // have to wait for network send + await delay(1) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.not.include(cid.toString()) + }) + + it('should send requested block presences to peer for blocks we don\'t have', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave, + sendDontHave: true + }] + } + }) + + // wait for network send + await pRetry(() => { + if (!components.network.sendMessage.called) { + throw new Error('Network message not sent') + } + }) + + const message = components.network.sendMessage.getCall(0).args[1] + + expect(message.blocks).to.be.empty('should not have sent blocks') + expect(message.blockPresences).to.have.lengthOf(1) + expect(message.blockPresences?.[0].cid).to.equalBytes(cid.bytes) + expect(message.blockPresences?.[0].type).to.equal(BlockPresenceType.DontHaveBlock, 'should have sent DontHaveBlock presence') + }) + + it('should remove wants when peer cancels', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + }) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.include(cid.toString()) + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + cancel: true + }] + } + }) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.not.include(cid.toString()) + }) + + it('should remove wantlist and ledger when peer disconnects', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3, 4]) + + // we have block + await components.blockstore.put(cid, block) + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + }) + + expect(wantLists.ledgerForPeer(remotePeer)).to.be.ok() + expect(wantLists.wantListForPeer(remotePeer)).to.be.ok() + + wantLists.peerDisconnected(remotePeer) + + expect(wantLists.ledgerForPeer(remotePeer)).to.be.undefined() + expect(wantLists.wantListForPeer(remotePeer)).to.be.undefined() + }) + + it('should return peers with want lists', async () => { + const remotePeer = await createEd25519PeerId() + + expect(wantLists.peers()).to.be.empty() + + await wantLists.messageReceived(remotePeer, { + blocks: [{ + prefix: Uint8Array.from([0, 1, 2, 3, 4]), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + prefix: Uint8Array.from([0, 1, 2]), + data: Uint8Array.from([0, 1, 2]) + }], + blockPresences: [], + pendingBytes: 0 + }) + + expect(wantLists.peers().map(p => p.toString())).to.include(remotePeer.toString()) + }) + + it('should send requested blocks to peer when they are received', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3, 4]) + + await wantLists.messageReceived(remotePeer, { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + }) + + expect(wantLists.wantListForPeer(remotePeer)?.map(e => e.cid.toString())).to.include(cid.toString()) + + // now we have block + await components.blockstore.put(cid, block) + + // we received it + await wantLists.receivedBlock(cid, {}) + + // wait for network send + await pRetry(() => { + if (!components.network.sendMessage.called) { + throw new Error('Network message not sent') + } + }) + + const message = components.network.sendMessage.getCall(0).args[1] + + expect(message.blocks).to.have.lengthOf(1) + expect(message.blocks?.[0].data).to.equalBytes(block) + expect(message.blocks?.[0].prefix).to.equalBytes(ve([ + cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength + ])) + + // have to wait for network send + await delay(1) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())) + .to.not.include(cid.toString()) + + // should only have sent one message + await delay(100) + expect(components.network.sendMessage.callCount).to.equal(1) + }) +}) diff --git a/packages/bitswap/test/session.spec.ts b/packages/bitswap/test/session.spec.ts new file mode 100644 index 00000000..efb77043 --- /dev/null +++ b/packages/bitswap/test/session.spec.ts @@ -0,0 +1,60 @@ +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { createBitswapSession } from '../src/session.js' +import type { BitswapSession } from '../src/index.js' +import type { Network } from '../src/network.js' +import type { Notifications } from '../src/notifications.js' +import type { WantList } from '../src/want-list.js' + +interface StubbedBitswapSessionComponents { + notifications: StubbedInstance + network: StubbedInstance + wantList: StubbedInstance +} + +describe('session', () => { + let components: StubbedBitswapSessionComponents + let session: BitswapSession + let cid: CID + + beforeEach(() => { + cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + components = { + notifications: stubInterface(), + network: stubInterface(), + wantList: stubInterface() + } + + session = createBitswapSession({ + ...components, + logger: defaultLogger() + }, { + root: cid + }) + }) + + it('should only query session peers', async () => { + const peerId = await createEd25519PeerId() + const data = new Uint8Array([0, 1, 2, 3, 4]) + + session.peers.add(peerId) + + components.notifications.wantBlock.resolves(data) + + const p = session.want(cid) + + expect(components.wantList.wantBlocks.called).to.be.true() + expect(components.wantList.wantBlocks.getCall(0)).to.have.nested.property('args[1].session', session.peers) + + await expect(p).to.eventually.deep.equal(data) + }) + + it('should throw when wanting from an empty session', async () => { + await expect(session.want(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_SESSION_PEERS') + }) +}) diff --git a/packages/bitswap/test/stats.spec.ts b/packages/bitswap/test/stats.spec.ts new file mode 100644 index 00000000..bd895295 --- /dev/null +++ b/packages/bitswap/test/stats.spec.ts @@ -0,0 +1,104 @@ +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { Stats } from '../src/stats.js' +import type { MetricGroup, Metrics } from '@libp2p/interface' + +interface StubbedStatsComponents { + metrics: StubbedInstance +} + +describe('stats', () => { + let stats: Stats + let components: StubbedStatsComponents + let metricGroup: StubbedInstance + + beforeEach(() => { + components = { + metrics: stubInterface() + } + + metricGroup = stubInterface() + + // @ts-expect-error tsc does not select correct method overload sig + components.metrics.registerMetricGroup.returns(metricGroup) + + stats = new Stats(components) + }) + + it('should update global blocks received', () => { + stats.updateBlocksReceived(1) + + expect(metricGroup.increment.calledWith({ + global: 1 + })).to.be.true() + }) + + it('should update blocks received from a peer', async () => { + const peerId = await createEd25519PeerId() + + stats.updateBlocksReceived(1, peerId) + + expect(metricGroup.increment.calledWith({ + global: 1, + [peerId.toString()]: 1 + })).to.be.true() + }) + + it('should update global duplicate blocks received', () => { + stats.updateDuplicateBlocksReceived(1) + + expect(metricGroup.increment.calledWith({ + global: 1 + })).to.be.true() + }) + + it('should update duplicate blocks received from a peer', async () => { + const peerId = await createEd25519PeerId() + + stats.updateDuplicateBlocksReceived(1, peerId) + + expect(metricGroup.increment.calledWith({ + global: 1, + [peerId.toString()]: 1 + })).to.be.true() + }) + + it('should update global data received', () => { + stats.updateDataReceived(1) + + expect(metricGroup.increment.calledWith({ + global: 1 + })).to.be.true() + }) + + it('should update data received from a peer', async () => { + const peerId = await createEd25519PeerId() + + stats.updateDataReceived(1, peerId) + + expect(metricGroup.increment.calledWith({ + global: 1, + [peerId.toString()]: 1 + })).to.be.true() + }) + + it('should update global duplicate data received', () => { + stats.updateDuplicateDataReceived(1) + + expect(metricGroup.increment.calledWith({ + global: 1 + })).to.be.true() + }) + + it('should update duplicate data received from a peer', async () => { + const peerId = await createEd25519PeerId() + + stats.updateDuplicateDataReceived(1, peerId) + + expect(metricGroup.increment.calledWith({ + global: 1, + [peerId.toString()]: 1 + })).to.be.true() + }) +}) diff --git a/packages/bitswap/test/want-list.spec.ts b/packages/bitswap/test/want-list.spec.ts new file mode 100644 index 00000000..18c4d605 --- /dev/null +++ b/packages/bitswap/test/want-list.spec.ts @@ -0,0 +1,126 @@ +import { defaultLogger } from '@libp2p/logger' +import { PeerSet } from '@libp2p/peer-collections' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { type Network } from '../src/network.js' +import { WantType } from '../src/pb/message.js' +import { WantList } from '../src/want-list.js' + +interface StubbedWantListComponents { + network: StubbedInstance +} + +describe('wantlist', () => { + let wantList: WantList + let components: StubbedWantListComponents + + beforeEach(() => { + components = { + network: stubInterface() + } + + wantList = new WantList({ + ...components, + logger: defaultLogger() + }) + + wantList.start() + }) + + afterEach(() => { + if (wantList != null) { + wantList.stop() + } + }) + + it('should add peers to peer list on connect', async () => { + const peerId = await createEd25519PeerId() + + await wantList.connected(peerId) + + expect(wantList.peers.has(peerId)).to.be.true() + }) + + it('should remove peers to peer list on disconnect', async () => { + const peerId = await createEd25519PeerId() + + await wantList.connected(peerId) + + expect(wantList.peers.has(peerId)).to.be.true() + + wantList.disconnected(peerId) + + expect(wantList.peers.has(peerId)).to.be.false() + }) + + it('should want blocks', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + await wantList.connected(peerId) + + components.network.sendMessage.withArgs(peerId).resolves() + await wantList.wantBlocks([cid]) + + const sentToPeer = components.network.sendMessage.getCall(0).args[0] + expect(sentToPeer.toString()).equal(peerId.toString()) + + const sentMessage = components.network.sendMessage.getCall(0).args[1] + expect(sentMessage).to.have.nested.property('wantlist.full', false) + expect(sentMessage).to.have.deep.nested.property('wantlist.entries[0].cid', cid.bytes) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].wantType', WantType.WantBlock) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].cancel', false) + }) + + it('should cancel wants', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + components.network.sendMessage.withArgs(peerId).resolves() + + await wantList.connected(peerId) + + await wantList.wantBlocks([cid]) + + expect([...wantList.wants.values()].find(want => want.cid.equals(cid))).to.be.ok() + + // now cancel + await wantList.cancelWants([cid]) + + const sentToPeer = components.network.sendMessage.getCall(1).args[0] + expect(sentToPeer.toString()).equal(peerId.toString()) + + const sentMessage = components.network.sendMessage.getCall(1).args[1] + expect(sentMessage).to.have.nested.property('wantlist.full', false) + expect(sentMessage).to.have.deep.nested.property('wantlist.entries[0].cid').that.equalBytes(cid.bytes) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].wantType', WantType.WantBlock) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].cancel', true) + + expect([...wantList.wants.values()].find(want => want.cid.equals(cid))).to.be.undefined() + }) + + it('should not send session block wants to non-session peers', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const sessionPeer = await createEd25519PeerId() + const nonSessionPeer = await createEd25519PeerId() + + await wantList.connected(sessionPeer) + await wantList.connected(nonSessionPeer) + + await wantList.wantBlocks([cid], { + session: new PeerSet([sessionPeer]) + }) + + expect(components.network.sendMessage.callCount).to.equal(1) + + const sentToPeer = components.network.sendMessage.getCall(0).args[0] + expect(sentToPeer.toString()).equal(sessionPeer.toString()) + + const sentMessage = components.network.sendMessage.getCall(0).args[1] + expect(sentMessage).to.have.nested.property('wantlist.full', false) + expect(sentMessage).to.have.deep.nested.property('wantlist.entries[0].cid', cid.bytes) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].wantType', WantType.WantBlock) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].cancel', false) + }) +}) diff --git a/packages/bitswap/tsconfig.json b/packages/bitswap/tsconfig.json new file mode 100644 index 00000000..4c0bdf77 --- /dev/null +++ b/packages/bitswap/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../interface" + } + ] +} diff --git a/packages/bitswap/typedoc.json b/packages/bitswap/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/bitswap/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/block-brokers/package.json b/packages/block-brokers/package.json index d51aef04..1e903958 100644 --- a/packages/block-brokers/package.json +++ b/packages/block-brokers/package.json @@ -53,6 +53,7 @@ "test:electron-main": "aegir test -t electron-main" }, "dependencies": { + "@helia/bitswap": "^0.0.0", "@helia/interface": "^4.0.0", "@libp2p/interface": "^1.1.2", "interface-blockstore": "^5.2.9", diff --git a/packages/block-brokers/src/bitswap.ts b/packages/block-brokers/src/bitswap.ts index dfe79ff1..939e70cc 100644 --- a/packages/block-brokers/src/bitswap.ts +++ b/packages/block-brokers/src/bitswap.ts @@ -1,8 +1,8 @@ -import { createBitswap } from 'ipfs-bitswap' -import type { BlockAnnounceOptions, BlockBroker, BlockRetrievalOptions } from '@helia/interface/blocks' -import type { Libp2p, Startable } from '@libp2p/interface' +import { createBitswap } from '@helia/bitswap' +import type { BitswapOptions, Bitswap, BitswapWantBlockProgressEvents, BitswapNotifyProgressEvents } from '@helia/bitswap' +import type { BlockAnnounceOptions, BlockBroker, BlockRetrievalOptions, CreateSessionOptions, Routing } from '@helia/interface' +import type { Libp2p, Startable, ComponentLogger } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' -import type { Bitswap, BitswapNotifyProgressEvents, BitswapOptions, BitswapWantBlockProgressEvents } from 'ipfs-bitswap' import type { CID } from 'multiformats/cid' import type { MultihashHasher } from 'multiformats/hashes/interface' @@ -10,6 +10,8 @@ interface BitswapComponents { libp2p: Libp2p blockstore: Blockstore hashers: Record + routing: Routing + logger: ComponentLogger } export interface BitswapInit extends BitswapOptions { @@ -21,9 +23,9 @@ class BitswapBlockBroker implements BlockBroker> => { let hasher: MultihashHasher | undefined @@ -69,6 +71,20 @@ class BitswapBlockBroker implements BlockBroker = {}): Promise { return this.bitswap.want(cid, options) } + + async createSession (root: CID, options?: CreateSessionOptions): Promise> { + const session = await this.bitswap.createSession(root, options) + + return { + announce: async (cid, block, options) => { + await this.bitswap.notify(cid, block, options) + }, + + retrieve: async (cid, options) => { + return session.want(cid, options) + } + } + } } /** diff --git a/packages/block-brokers/tsconfig.json b/packages/block-brokers/tsconfig.json index 4c0bdf77..26c90c06 100644 --- a/packages/block-brokers/tsconfig.json +++ b/packages/block-brokers/tsconfig.json @@ -8,6 +8,9 @@ "test" ], "references": [ + { + "path": "../bitswap" + }, { "path": "../interface" } From 73ab5f98a9dec48188a6bf6a868cd7fdd33a24f7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 31 Jan 2024 08:00:16 +0100 Subject: [PATCH 08/27] chore: fix up tests --- packages/bitswap/src/bitswap.ts | 127 +++++++++++++------------- packages/bitswap/src/index.ts | 11 ++- packages/bitswap/test/bitswap.spec.ts | 6 +- packages/bitswap/test/network.spec.ts | 2 +- 4 files changed, 79 insertions(+), 67 deletions(-) diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts index 85b0c15f..afed5aaf 100644 --- a/packages/bitswap/src/bitswap.ts +++ b/packages/bitswap/src/bitswap.ts @@ -3,6 +3,7 @@ import { setMaxListeners } from '@libp2p/interface' import { PeerSet } from '@libp2p/peer-collections' import { PeerQueue } from '@libp2p/utils/peer-queue' import { anySignal } from 'any-signal' +import drain from 'it-drain' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' import pDefer from 'p-defer' @@ -103,40 +104,58 @@ export class Bitswap implements BitswapInterface { * handle messages received through the network */ async _receiveMessage (peerId: PeerId, message: BitswapMessage): Promise { - // hash all incoming blocks - const received = await Promise.all( - message.blocks - .filter(block => block.prefix != null && block.data != null) - .map(async block => { - const values = vd(block.prefix) - const cidVersion = values[0] - const multicodec = values[1] - const hashAlg = values[2] - // const hashLen = values[3] // We haven't need to use this so far - - const hasher = hashAlg === sha256.code ? sha256 : await this.hashLoader?.getHasher(hashAlg) - - if (hasher == null) { - throw new CodeError('Unknown hash algorithm', 'ERR_UNKNOWN_HASH_ALG') - } + // CIDs we received + const received: CID[] = [] + + // process all incoming blocks + const self = this + await drain(this.blockstore.putMany(async function * () { + for (const block of message.blocks) { + if (block.prefix == null || block.data == null) { + continue + } - const hash = await hasher.digest(block.data) - const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash) - const wasWanted = this.notifications.listenerCount(receivedBlockEvent(cid)) > 0 + self.log('received block') + const values = vd(block.prefix) + const cidVersion = values[0] + const multicodec = values[1] + const hashAlg = values[2] + // const hashLen = values[3] // We haven't need to use this so far - return { wasWanted, cid, data: block.data } - }) - ) + const hasher = hashAlg === sha256.code ? sha256 : await self.hashLoader?.getHasher(hashAlg) + + if (hasher == null) { + self.log.error('unknown hash algorithm', hashAlg) + continue + } + + const hash = await hasher.digest(block.data) + const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash) + const wasWanted = self.notifications.listenerCount(receivedBlockEvent(cid)) > 0 + received.push(cid) + + const has = await self.blockstore.has(cid) + + self._updateReceiveCounters(peerId, block.data, has) + + if (!wasWanted) { + continue + } + + if (!has) { + yield { cid, block: block.data } + } + + self.notifications.receivedBlock(cid, block.data, peerId) + } + }())) // quickly send out cancels, reduces chances of duplicate block receives if (received.length > 0) { - this.wantList.cancelWants( - received - .filter(({ wasWanted }) => wasWanted) - .map(({ cid }) => cid) - ).catch(err => { - this.log.error('error sending block cancels', err) - }) + this.wantList.cancelWants(received) + .catch(err => { + this.log.error('error sending block cancels', err) + }) } // notify sessions of block haves/don't haves @@ -150,43 +169,15 @@ export class Bitswap implements BitswapInterface { } } - await Promise.all( - received.map( - async ({ cid, wasWanted, data }) => { - await this._handleReceivedBlock(peerId, cid, data, wasWanted) - this.notifications.receivedBlock(cid, data, peerId) - } - ) - ) - try { - // Note: this allows the engine to respond to any wants in the message. - // Processing of the blocks in the message happens below, after the - // blocks have been added to the blockstore. + // Respond to any wants in the message await this.peerWantLists.messageReceived(peerId, message) } catch (err) { - // Log instead of throwing an error so as to process as much as - // possible of the message. Currently `messageReceived` does not - // throw any errors, but this could change in the future. this.log('failed to receive message from %p', peerId, message) } } - private async _handleReceivedBlock (peerId: PeerId, cid: CID, data: Uint8Array, wasWanted: boolean): Promise { - this.log('received block') - - const has = await this.blockstore.has(cid) - - this._updateReceiveCounters(peerId, cid, data, has) - - if (!wasWanted || has) { - return - } - - await this.blockstore.put(cid, data) - } - - _updateReceiveCounters (peerId: PeerId, cid: CID, data: Uint8Array, exists: boolean): void { + _updateReceiveCounters (peerId: PeerId, data: Uint8Array, exists: boolean): void { this.stats.updateBlocksReceived(1, peerId) this.stats.updateDataReceived(data.byteLength, peerId) @@ -227,9 +218,9 @@ export class Bitswap implements BitswapInterface { this.notifications.removeListener(receivedBlockEvent(root), receivedBlockListener) this.notifications.removeListener(haveEvent(root), haveBlockListener) this.notifications.removeListener(doNotHaveEvent(root), doNotHaveBlockListener) - } - queue.clear() + queue.clear() + } }) const queriedPeers = new PeerSet() @@ -245,6 +236,14 @@ export class Bitswap implements BitswapInterface { if (session.peers.size === minProviders) { deferred.resolve(session) } + + if (session.peers.size === maxProviders) { + this.notifications.removeListener(receivedBlockEvent(root), receivedBlockListener) + this.notifications.removeListener(haveEvent(root), haveBlockListener) + this.notifications.removeListener(doNotHaveEvent(root), doNotHaveBlockListener) + + queue.clear() + } } const haveBlockListener: HaveBlockListener = (peer): void => { this.log('adding %p to session after receiving HAVE_BLOCK', peer) @@ -259,6 +258,8 @@ export class Bitswap implements BitswapInterface { this.notifications.removeListener(receivedBlockEvent(root), receivedBlockListener) this.notifications.removeListener(haveEvent(root), haveBlockListener) this.notifications.removeListener(doNotHaveEvent(root), doNotHaveBlockListener) + + queue.clear() } } const doNotHaveBlockListener: DoNotHaveBlockListener = (peer) => { @@ -312,6 +313,10 @@ export class Bitswap implements BitswapInterface { // find network providers too but do not wait for the query to complete void Promise.resolve().then(async () => { + if (options?.queryRoutingPeers === false) { + return + } + let providers = 0 for await (const provider of this.network.findProviders(root, options)) { diff --git a/packages/bitswap/src/index.ts b/packages/bitswap/src/index.ts index e37004b1..9fc0aa78 100644 --- a/packages/bitswap/src/index.ts +++ b/packages/bitswap/src/index.ts @@ -68,7 +68,7 @@ export interface CreateSessionOptions extends AbortOptions, ProgressOptions { expect(providers[1].id.equals(components.libp2p.dialProtocol.getCall(2).args[0].toString())).to.be.true() expect(providers[2].id.equals(components.libp2p.dialProtocol.getCall(3).args[0].toString())).to.be.true() - // one current peer and providers 1-4 - expect(components.libp2p.dialProtocol.callCount).to.equal(5) - // should have stopped at DEFAULT_MAX_PROVIDERS_PER_REQUEST expect(session.peers.size).to.equal(DEFAULT_MAX_PROVIDERS_PER_REQUEST) }) @@ -436,6 +433,9 @@ function stubPeerResponse (libp2p: StubbedInstance, peerId: PeerId, resp const pbstr = pbStream(localStream).pb(BitswapMessage) void pbstr.read().then(async message => { + // simulate network latency + await delay(10) + // after reading message from remote, open a new stream on the remote and // send the response const [localDuplex, remoteDuplex] = duplexPair() diff --git a/packages/bitswap/test/network.spec.ts b/packages/bitswap/test/network.spec.ts index 0e8cdacc..ff93c68c 100644 --- a/packages/bitswap/test/network.spec.ts +++ b/packages/bitswap/test/network.spec.ts @@ -398,7 +398,7 @@ describe('network', () => { prefix: cidToPrefix(cid1), data: Uint8Array.from([0, 1, 2, 3, 4]) }] - }) + }).catch(() => {}) // send two messages while the queue is blocked void network.sendMessage(peerId, messageA) From 0c28d660e6373c5f603985c42de33a0f1e03f50b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 8 Feb 2024 09:55:31 +0100 Subject: [PATCH 09/27] chore: create block brokers last to access components --- packages/utils/src/index.ts | 27 ++++++++++--------- packages/utils/src/utils/networked-storage.ts | 26 +++++++++--------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 7890f879..65f29454 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -112,6 +112,7 @@ interface Components { dagWalkers: Record logger: ComponentLogger blockBrokers: BlockBroker[] + routing: Routing } export class Helia implements HeliaInterface { @@ -130,6 +131,7 @@ export class Helia implements HeliaInterface { this.hashers = defaultHashers(init.hashers) this.dagWalkers = defaultDagWalkers(init.dagWalkers) + // @ts-expect-error routing is not set const components: Components = { blockstore: init.blockstore, datastore: init.datastore, @@ -140,19 +142,7 @@ export class Helia implements HeliaInterface { ...(init.components ?? {}) } - components.blockBrokers = init.blockBrokers.map((fn) => { - return fn(components) - }) - - const networkedStorage = new NetworkedStorage(components) - - this.pins = new PinsImpl(init.datastore, networkedStorage, this.dagWalkers) - - this.blockstore = new BlockStorage(networkedStorage, this.pins, { - holdGcLock: init.holdGcLock ?? true - }) - this.datastore = init.datastore - this.routing = new RoutingClass(components, { + this.routing = components.routing = new RoutingClass(components, { routers: (init.routers ?? []).flatMap((router: any) => { // if the router itself is a router const routers = [ @@ -172,6 +162,17 @@ export class Helia implements HeliaInterface { return routers }) }) + + const networkedStorage = new NetworkedStorage(components) + this.pins = new PinsImpl(init.datastore, networkedStorage, this.dagWalkers) + this.blockstore = new BlockStorage(networkedStorage, this.pins, { + holdGcLock: init.holdGcLock ?? true + }) + this.datastore = init.datastore + + components.blockBrokers = init.blockBrokers.map((fn) => { + return fn(components) + }) } async start (): Promise { diff --git a/packages/utils/src/utils/networked-storage.ts b/packages/utils/src/utils/networked-storage.ts index 57b58c78..0a59cf01 100644 --- a/packages/utils/src/utils/networked-storage.ts +++ b/packages/utils/src/utils/networked-storage.ts @@ -22,8 +22,8 @@ export interface GetOptions extends AbortOptions { export interface NetworkedStorageComponents { blockstore: Blockstore logger: ComponentLogger - blockBrokers?: BlockBroker[] - hashers?: Record + blockBrokers: BlockBroker[] + hashers: Record } /** @@ -32,11 +32,11 @@ export interface NetworkedStorageComponents { */ export class NetworkedStorage implements Blocks, Startable { private readonly child: Blockstore - private readonly blockBrokers: BlockBroker[] private readonly hashers: Record private started: boolean private readonly log: Logger private readonly logger: ComponentLogger + private readonly components: NetworkedStorageComponents /** * Create a new BlockStorage @@ -45,7 +45,7 @@ export class NetworkedStorage implements Blocks, Startable { this.log = components.logger.forComponent(`helia:networked-storage${init.root == null ? '' : `:${init.root}`}`) this.logger = components.logger this.child = components.blockstore - this.blockBrokers = components.blockBrokers ?? [] + this.components = components this.hashers = components.hashers ?? {} this.started = false } @@ -55,12 +55,12 @@ export class NetworkedStorage implements Blocks, Startable { } async start (): Promise { - await start(this.child, ...this.blockBrokers) + await start(this.child, ...this.components.blockBrokers) this.started = true } async stop (): Promise { - await stop(this.child, ...this.blockBrokers) + await stop(this.child, ...this.components.blockBrokers) this.started = false } @@ -80,7 +80,7 @@ export class NetworkedStorage implements Blocks, Startable { options.onProgress?.(new CustomProgressEvent('blocks:put:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) + this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid)) @@ -105,7 +105,7 @@ export class NetworkedStorage implements Blocks, Startable { const notifyEach = forEach(missingBlocks, async ({ cid, block }): Promise => { options.onProgress?.(new CustomProgressEvent('blocks:put-many:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) + this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) }) @@ -120,7 +120,7 @@ export class NetworkedStorage implements Blocks, Startable { if (options.offline !== true && !(await this.child.has(cid))) { // we do not have the block locally, get it from a block provider options.onProgress?.(new CustomProgressEvent('blocks:get:providers:get', cid)) - const block = await raceBlockRetrievers(cid, this.blockBrokers, this.hashers[cid.multihash.code], { + const block = await raceBlockRetrievers(cid, this.components.blockBrokers, this.hashers[cid.multihash.code], { ...options, log: this.log }) @@ -130,7 +130,7 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) + this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) return block @@ -151,7 +151,7 @@ export class NetworkedStorage implements Blocks, Startable { if (options.offline !== true && !(await this.child.has(cid))) { // we do not have the block locally, get it from a block provider options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:get', cid)) - const block = await raceBlockRetrievers(cid, this.blockBrokers, this.hashers[cid.multihash.code], { + const block = await raceBlockRetrievers(cid, this.components.blockBrokers, this.hashers[cid.multihash.code], { ...options, log: this.log }) @@ -161,7 +161,7 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) + this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) } })) @@ -198,7 +198,7 @@ export class NetworkedStorage implements Blocks, Startable { } async createSession (root: CID, options?: AbortOptions & ProgressOptions): Promise { - const blockBrokers = await Promise.all(this.blockBrokers.map(async broker => { + const blockBrokers = await Promise.all(this.components.blockBrokers.map(async broker => { if (broker.createSession == null) { return broker } From e9bbbe63c38aa006a227fa0bc170d4f462df9b8d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 8 Feb 2024 09:55:31 +0100 Subject: [PATCH 10/27] chore: create block brokers last to access components --- packages/utils/src/index.ts | 27 ++++++++++--------- packages/utils/src/utils/networked-storage.ts | 26 +++++++++--------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 7890f879..65f29454 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -112,6 +112,7 @@ interface Components { dagWalkers: Record logger: ComponentLogger blockBrokers: BlockBroker[] + routing: Routing } export class Helia implements HeliaInterface { @@ -130,6 +131,7 @@ export class Helia implements HeliaInterface { this.hashers = defaultHashers(init.hashers) this.dagWalkers = defaultDagWalkers(init.dagWalkers) + // @ts-expect-error routing is not set const components: Components = { blockstore: init.blockstore, datastore: init.datastore, @@ -140,19 +142,7 @@ export class Helia implements HeliaInterface { ...(init.components ?? {}) } - components.blockBrokers = init.blockBrokers.map((fn) => { - return fn(components) - }) - - const networkedStorage = new NetworkedStorage(components) - - this.pins = new PinsImpl(init.datastore, networkedStorage, this.dagWalkers) - - this.blockstore = new BlockStorage(networkedStorage, this.pins, { - holdGcLock: init.holdGcLock ?? true - }) - this.datastore = init.datastore - this.routing = new RoutingClass(components, { + this.routing = components.routing = new RoutingClass(components, { routers: (init.routers ?? []).flatMap((router: any) => { // if the router itself is a router const routers = [ @@ -172,6 +162,17 @@ export class Helia implements HeliaInterface { return routers }) }) + + const networkedStorage = new NetworkedStorage(components) + this.pins = new PinsImpl(init.datastore, networkedStorage, this.dagWalkers) + this.blockstore = new BlockStorage(networkedStorage, this.pins, { + holdGcLock: init.holdGcLock ?? true + }) + this.datastore = init.datastore + + components.blockBrokers = init.blockBrokers.map((fn) => { + return fn(components) + }) } async start (): Promise { diff --git a/packages/utils/src/utils/networked-storage.ts b/packages/utils/src/utils/networked-storage.ts index 57b58c78..0a59cf01 100644 --- a/packages/utils/src/utils/networked-storage.ts +++ b/packages/utils/src/utils/networked-storage.ts @@ -22,8 +22,8 @@ export interface GetOptions extends AbortOptions { export interface NetworkedStorageComponents { blockstore: Blockstore logger: ComponentLogger - blockBrokers?: BlockBroker[] - hashers?: Record + blockBrokers: BlockBroker[] + hashers: Record } /** @@ -32,11 +32,11 @@ export interface NetworkedStorageComponents { */ export class NetworkedStorage implements Blocks, Startable { private readonly child: Blockstore - private readonly blockBrokers: BlockBroker[] private readonly hashers: Record private started: boolean private readonly log: Logger private readonly logger: ComponentLogger + private readonly components: NetworkedStorageComponents /** * Create a new BlockStorage @@ -45,7 +45,7 @@ export class NetworkedStorage implements Blocks, Startable { this.log = components.logger.forComponent(`helia:networked-storage${init.root == null ? '' : `:${init.root}`}`) this.logger = components.logger this.child = components.blockstore - this.blockBrokers = components.blockBrokers ?? [] + this.components = components this.hashers = components.hashers ?? {} this.started = false } @@ -55,12 +55,12 @@ export class NetworkedStorage implements Blocks, Startable { } async start (): Promise { - await start(this.child, ...this.blockBrokers) + await start(this.child, ...this.components.blockBrokers) this.started = true } async stop (): Promise { - await stop(this.child, ...this.blockBrokers) + await stop(this.child, ...this.components.blockBrokers) this.started = false } @@ -80,7 +80,7 @@ export class NetworkedStorage implements Blocks, Startable { options.onProgress?.(new CustomProgressEvent('blocks:put:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) + this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) options.onProgress?.(new CustomProgressEvent('blocks:put:blockstore:put', cid)) @@ -105,7 +105,7 @@ export class NetworkedStorage implements Blocks, Startable { const notifyEach = forEach(missingBlocks, async ({ cid, block }): Promise => { options.onProgress?.(new CustomProgressEvent('blocks:put-many:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) + this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) }) @@ -120,7 +120,7 @@ export class NetworkedStorage implements Blocks, Startable { if (options.offline !== true && !(await this.child.has(cid))) { // we do not have the block locally, get it from a block provider options.onProgress?.(new CustomProgressEvent('blocks:get:providers:get', cid)) - const block = await raceBlockRetrievers(cid, this.blockBrokers, this.hashers[cid.multihash.code], { + const block = await raceBlockRetrievers(cid, this.components.blockBrokers, this.hashers[cid.multihash.code], { ...options, log: this.log }) @@ -130,7 +130,7 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) + this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) return block @@ -151,7 +151,7 @@ export class NetworkedStorage implements Blocks, Startable { if (options.offline !== true && !(await this.child.has(cid))) { // we do not have the block locally, get it from a block provider options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:get', cid)) - const block = await raceBlockRetrievers(cid, this.blockBrokers, this.hashers[cid.multihash.code], { + const block = await raceBlockRetrievers(cid, this.components.blockBrokers, this.hashers[cid.multihash.code], { ...options, log: this.log }) @@ -161,7 +161,7 @@ export class NetworkedStorage implements Blocks, Startable { // notify other block providers of the new block options.onProgress?.(new CustomProgressEvent('blocks:get-many:providers:notify', cid)) await Promise.all( - this.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) + this.components.blockBrokers.map(async broker => broker.announce?.(cid, block, options)) ) } })) @@ -198,7 +198,7 @@ export class NetworkedStorage implements Blocks, Startable { } async createSession (root: CID, options?: AbortOptions & ProgressOptions): Promise { - const blockBrokers = await Promise.all(this.blockBrokers.map(async broker => { + const blockBrokers = await Promise.all(this.components.blockBrokers.map(async broker => { if (broker.createSession == null) { return broker } From 1dca655d56198f3d2a9f06c2521d67522d212a12 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 8 Feb 2024 10:52:46 +0100 Subject: [PATCH 11/27] chore: fix tests --- packages/bitswap/src/bitswap.ts | 2 +- packages/bitswap/src/index.ts | 2 +- packages/bitswap/src/network.ts | 2 +- packages/bitswap/src/peer-want-lists/index.ts | 12 +++++++++++- packages/bitswap/test/bitswap.spec.ts | 2 +- packages/bitswap/test/network.spec.ts | 2 +- packages/block-brokers/package.json | 1 - packages/block-brokers/src/bitswap.ts | 2 +- packages/interop/src/unixfs-bitswap.spec.ts | 8 ++------ 9 files changed, 19 insertions(+), 14 deletions(-) diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts index afed5aaf..917e8b20 100644 --- a/packages/bitswap/src/bitswap.ts +++ b/packages/bitswap/src/bitswap.ts @@ -56,7 +56,7 @@ export class Bitswap implements BitswapInterface { constructor (components: BitswapComponents, init: BitswapOptions = {}) { this.logger = components.logger - this.log = components.logger.forComponent('bitswap') + this.log = components.logger.forComponent('helia:bitswap') this.status = 'stopped' this.libp2p = components.libp2p this.blockstore = components.blockstore diff --git a/packages/bitswap/src/index.ts b/packages/bitswap/src/index.ts index 9fc0aa78..7c66883c 100644 --- a/packages/bitswap/src/index.ts +++ b/packages/bitswap/src/index.ts @@ -7,7 +7,7 @@ import { Bitswap as BitswapClass } from './bitswap.js' import type { BitswapNetworkNotifyProgressEvents, BitswapNetworkWantProgressEvents } from './network.js' import type { WantType } from './pb/message.js' -import type { Routing } from '@helia/interface' +import type { Routing } from '@helia/interface/routing' import type { Libp2p, AbortOptions, Startable, ComponentLogger, Metrics, PeerId } from '@libp2p/interface' import type { PeerSet } from '@libp2p/peer-collections' import type { Blockstore } from 'interface-blockstore' diff --git a/packages/bitswap/src/network.ts b/packages/bitswap/src/network.ts index d1834f91..c4a53e4e 100644 --- a/packages/bitswap/src/network.ts +++ b/packages/bitswap/src/network.ts @@ -18,7 +18,7 @@ import { BitswapMessage } from './pb/message.js' import type { WantOptions } from './bitswap.js' import type { MultihashHasherLoader } from './index.js' import type { Block, BlockPresence, WantlistEntry } from './pb/message.js' -import type { Provider, Routing } from '@helia/interface' +import type { Provider, Routing } from '@helia/interface/routing' import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, MetricGroup, ComponentLogger, Metrics } from '@libp2p/interface' import type { Logger } from '@libp2p/logger' import type { ProgressEvent, ProgressOptions } from 'progress-events' diff --git a/packages/bitswap/src/peer-want-lists/index.ts b/packages/bitswap/src/peer-want-lists/index.ts index 584ff5e9..692c4d91 100644 --- a/packages/bitswap/src/peer-want-lists/index.ts +++ b/packages/bitswap/src/peer-want-lists/index.ts @@ -6,7 +6,7 @@ import { Ledger } from './ledger.js' import type { BitswapNotifyProgressEvents, WantListEntry } from '../index.js' import type { Network } from '../network.js' import type { BitswapMessage } from '../pb/message.js' -import type { ComponentLogger, Metrics, PeerId } from '@libp2p/interface' +import type { ComponentLogger, Logger, Metrics, PeerId } from '@libp2p/interface' import type { PeerMap } from '@libp2p/peer-collections' import type { Blockstore } from 'interface-blockstore' import type { AbortOptions } from 'it-length-prefixed-stream' @@ -36,11 +36,13 @@ export class PeerWantLists { public network: Network public readonly ledgerMap: PeerMap private readonly maxSizeReplaceHasWithBlock?: number + private readonly log: Logger constructor (components: PeerWantListsComponents, init: PeerWantListsInit = {}) { this.blockstore = components.blockstore this.network = components.network this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock + this.log = components.logger.forComponent('helia:bitswap:peer-want-lists') this.ledgerMap = trackedPeerMap({ name: 'ipfs_bitswap_ledger_map', @@ -110,8 +112,16 @@ export class PeerWantLists { const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') if (entry.cancel === true) { + this.log('peer %p cancelled want of block for %c', peerId, cid) + ledger.wants.delete(cidStr) } else { + if (entry.wantType === WantType.WantHave) { + this.log('peer %p wanted block presence for %c', peerId, cid) + } else { + this.log('peer %p wanted block for %c', peerId, cid) + } + ledger.wants.set(cidStr, { cid, session: new PeerSet(), diff --git a/packages/bitswap/test/bitswap.spec.ts b/packages/bitswap/test/bitswap.spec.ts index 80068269..8ed9101f 100644 --- a/packages/bitswap/test/bitswap.spec.ts +++ b/packages/bitswap/test/bitswap.spec.ts @@ -17,7 +17,7 @@ import { Bitswap } from '../src/bitswap.js' import { DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY } from '../src/constants.js' import { BitswapMessage, BlockPresenceType } from '../src/pb/message.js' import { cidToPrefix } from '../src/utils/cid-prefix.js' -import type { Routing } from '@helia/interface' +import type { Routing } from '@helia/interface/routing' import type { Connection, Libp2p, PeerId } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { StubbedInstance } from 'sinon-ts' diff --git a/packages/bitswap/test/network.spec.ts b/packages/bitswap/test/network.spec.ts index ff93c68c..87f7fb7e 100644 --- a/packages/bitswap/test/network.spec.ts +++ b/packages/bitswap/test/network.spec.ts @@ -17,7 +17,7 @@ import { BITSWAP_120 } from '../src/constants.js' import { Network } from '../src/network.js' import { BitswapMessage, BlockPresenceType } from '../src/pb/message.js' import { cidToPrefix } from '../src/utils/cid-prefix.js' -import type { Routing } from '@helia/interface' +import type { Routing } from '@helia/interface/routing' import type { Connection, Libp2p, PeerId } from '@libp2p/interface' interface StubbedNetworkComponents { diff --git a/packages/block-brokers/package.json b/packages/block-brokers/package.json index 1e903958..d119a14c 100644 --- a/packages/block-brokers/package.json +++ b/packages/block-brokers/package.json @@ -57,7 +57,6 @@ "@helia/interface": "^4.0.0", "@libp2p/interface": "^1.1.2", "interface-blockstore": "^5.2.9", - "ipfs-bitswap": "^20.0.2", "multiformats": "^13.0.1", "progress-events": "^1.0.0" }, diff --git a/packages/block-brokers/src/bitswap.ts b/packages/block-brokers/src/bitswap.ts index 939e70cc..f2506ee4 100644 --- a/packages/block-brokers/src/bitswap.ts +++ b/packages/block-brokers/src/bitswap.ts @@ -65,7 +65,7 @@ class BitswapBlockBroker implements BlockBroker): Promise { - this.bitswap.notify(cid, block, options) + await this.bitswap.notify(cid, block, options) } async retrieve (cid: CID, options: BlockRetrievalOptions = {}): Promise { diff --git a/packages/interop/src/unixfs-bitswap.spec.ts b/packages/interop/src/unixfs-bitswap.spec.ts index e25206a0..6c6cc283 100644 --- a/packages/interop/src/unixfs-bitswap.spec.ts +++ b/packages/interop/src/unixfs-bitswap.spec.ts @@ -52,13 +52,9 @@ describe('@helia/unixfs - bitswap', () => { const cid = await unixFs.addFile(candidate) - const output: Uint8Array[] = [] + const bytes = await toBuffer(kubo.api.cat(CID.parse(cid.toString()))) - for await (const b of kubo.api.cat(cid)) { - output.push(b) - } - - expect(toBuffer(output)).to.equalBytes(toBuffer(input)) + expect(bytes).to.equalBytes(toBuffer(input)) }) it('should add a large file to kubo and fetch it from helia', async () => { From d0610c9c665c7e87733792fa26f87045587c5e95 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Feb 2024 09:39:26 +0100 Subject: [PATCH 12/27] chore: do not dedupe at the routing level as different impls return different metadata --- packages/interface/src/blocks.ts | 23 ++++++++++++++++++++++- packages/utils/src/routing.ts | 19 ------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index fd283dd6..078bad50 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -88,8 +88,29 @@ export interface CreateSessionOptions router.findProviders(key, options)) @@ -50,13 +47,6 @@ export class Routing implements RoutingInterface, Startable { continue } - // deduplicate peers - if (seen.has(peer.id)) { - continue - } - - seen.add(peer.id) - yield peer } } @@ -142,8 +132,6 @@ export class Routing implements RoutingInterface, Startable { throw new CodeError('No peer routers available', 'ERR_NO_ROUTERS_AVAILABLE') } - const seen = new PeerSet() - for await (const peer of merge( ...supports(this.routers, 'getClosestPeers') .map(router => router.getClosestPeers(key, options)) @@ -152,13 +140,6 @@ export class Routing implements RoutingInterface, Startable { continue } - // deduplicate peers - if (seen.has(peer.id)) { - continue - } - - seen.add(peer.id) - yield peer } } From 9aeded1ae2c3ada8a19148031f493bd42e0db981 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Feb 2024 10:33:39 +0100 Subject: [PATCH 13/27] chore: add defaults to interface --- packages/interface/src/blocks.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index 078bad50..57200774 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -137,3 +137,7 @@ export interface BlockBroker): Promise> } + +export const DEFAULT_MIN_SESSION_PROVIDERS = 1 +export const DEFAULT_MAX_SESSION_PROVIDERS = 5 +export const DEFAULT_SESSION_QUERY_CONCURRENCY = 5 From 8e6051a4cfe15b2e606cd5816a055d8284209143 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Feb 2024 10:34:33 +0100 Subject: [PATCH 14/27] chore: remove unused dep --- packages/utils/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/utils/package.json b/packages/utils/package.json index 937c4469..5b9adec2 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -59,7 +59,6 @@ "@ipld/dag-pb": "^4.0.8", "@libp2p/interface": "^1.1.2", "@libp2p/logger": "^4.0.5", - "@libp2p/peer-collections": "^5.1.5", "@libp2p/utils": "^5.2.3", "any-signal": "^4.1.1", "cborg": "^4.0.8", From ca1c45827b118101dd2fb2e6fce3cc12017a8bd2 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Feb 2024 12:25:54 +0100 Subject: [PATCH 15/27] chore: update constants --- packages/interface/src/blocks.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index 57200774..5511b7b0 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -118,7 +118,7 @@ export interface CreateSessionOptions = ProgressEvent, AnnounceProgressEvents extends ProgressEvent = ProgressEvent> { @@ -138,6 +138,7 @@ export interface BlockBroker): Promise> } -export const DEFAULT_MIN_SESSION_PROVIDERS = 1 -export const DEFAULT_MAX_SESSION_PROVIDERS = 5 +export const DEFAULT_SESSION_MIN_PROVIDERS = 1 +export const DEFAULT_SESSION_MAX_PROVIDERS = 5 export const DEFAULT_SESSION_QUERY_CONCURRENCY = 5 +export const DEFAULT_SESSION_PROVIDER_QUERY_TIMEOUT = 5000 From 8cd06ce3fa49ed9c316c2ca0cadc6fcc55337e0b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 9 Feb 2024 12:31:30 +0100 Subject: [PATCH 16/27] chore: reuse constants --- packages/bitswap/src/bitswap.ts | 11 ++++++----- packages/bitswap/src/constants.ts | 3 --- packages/bitswap/src/index.ts | 28 ++------------------------- packages/bitswap/test/bitswap.spec.ts | 10 +++++----- 4 files changed, 13 insertions(+), 39 deletions(-) diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts index 917e8b20..204eb3ac 100644 --- a/packages/bitswap/src/bitswap.ts +++ b/packages/bitswap/src/bitswap.ts @@ -1,4 +1,5 @@ /* eslint-disable no-loop-func */ +import { DEFAULT_SESSION_MAX_PROVIDERS, DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_QUERY_CONCURRENCY } from '@helia/interface' import { setMaxListeners } from '@libp2p/interface' import { PeerSet } from '@libp2p/peer-collections' import { PeerQueue } from '@libp2p/utils/peer-queue' @@ -9,7 +10,7 @@ import { sha256 } from 'multiformats/hashes/sha2' import pDefer from 'p-defer' import { CodeError } from 'protons-runtime' import { raceSignal } from 'race-signal' -import { DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY, DEFAULT_SESSION_QUERY_CONCURRENCY, DEFAULT_SESSION_ROOT_PRIORITY } from './constants.js' +import { DEFAULT_SESSION_ROOT_PRIORITY } from './constants.js' import { Network } from './network.js' import { Notifications, receivedBlockEvent, type ReceivedBlockListener, type HaveBlockListener, haveEvent, type DoNotHaveBlockListener, doNotHaveEvent } from './notifications.js' import { BlockPresenceType, WantType } from './pb/message.js' @@ -18,7 +19,7 @@ import { createBitswapSession } from './session.js' import { Stats } from './stats.js' import vd from './utils/varint-decoder.js' import { WantList } from './want-list.js' -import type { BitswapOptions, Bitswap as BitswapInterface, MultihashHasherLoader, BitswapWantProgressEvents, BitswapNotifyProgressEvents, BitswapSession, WantListEntry, CreateSessionOptions, BitswapComponents } from './index.js' +import type { BitswapOptions, Bitswap as BitswapInterface, MultihashHasherLoader, BitswapWantProgressEvents, BitswapNotifyProgressEvents, BitswapSession, WantListEntry, BitswapComponents, CreateBitswapSessionOptions } from './index.js' import type { BitswapMessage } from './pb/message.js' import type { ComponentLogger, Libp2p, PeerId } from '@libp2p/interface' import type { Logger } from '@libp2p/logger' @@ -187,9 +188,9 @@ export class Bitswap implements BitswapInterface { } } - async createSession (root: CID, options?: CreateSessionOptions): Promise { - const minProviders = options?.minProviders ?? DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY - const maxProviders = options?.maxProviders ?? DEFAULT_MAX_PROVIDERS_PER_REQUEST + async createSession (root: CID, options?: CreateBitswapSessionOptions): Promise { + const minProviders = options?.minProviders ?? DEFAULT_SESSION_MIN_PROVIDERS + const maxProviders = options?.maxProviders ?? DEFAULT_SESSION_MAX_PROVIDERS // normalize to v1 CID root = root.toV1() diff --git a/packages/bitswap/src/constants.ts b/packages/bitswap/src/constants.ts index ae8c86d3..3671f037 100644 --- a/packages/bitswap/src/constants.ts +++ b/packages/bitswap/src/constants.ts @@ -1,6 +1,4 @@ export const BITSWAP_120 = '/ipfs/bitswap/1.2.0' -export const DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY = 1 -export const DEFAULT_MAX_PROVIDERS_PER_REQUEST = 3 export const DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK = 1024 export const DEFAULT_MAX_INBOUND_STREAMS = 1024 export const DEFAULT_MAX_OUTBOUND_STREAMS = 1024 @@ -9,4 +7,3 @@ export const DEFAULT_MESSAGE_SEND_TIMEOUT = 5000 export const DEFAULT_MESSAGE_SEND_CONCURRENCY = 50 export const DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS = false export const DEFAULT_SESSION_ROOT_PRIORITY = 1 -export const DEFAULT_SESSION_QUERY_CONCURRENCY = 5 diff --git a/packages/bitswap/src/index.ts b/packages/bitswap/src/index.ts index 7c66883c..9c441a8b 100644 --- a/packages/bitswap/src/index.ts +++ b/packages/bitswap/src/index.ts @@ -7,6 +7,7 @@ import { Bitswap as BitswapClass } from './bitswap.js' import type { BitswapNetworkNotifyProgressEvents, BitswapNetworkWantProgressEvents } from './network.js' import type { WantType } from './pb/message.js' +import type { CreateSessionOptions } from '@helia/interface' import type { Routing } from '@helia/interface/routing' import type { Libp2p, AbortOptions, Startable, ComponentLogger, Metrics, PeerId } from '@libp2p/interface' import type { PeerSet } from '@libp2p/peer-collections' @@ -56,24 +57,7 @@ export interface WantListEntry { sentDontHave?: boolean } -export interface CreateSessionOptions extends AbortOptions, ProgressOptions { - /** - * The session will be ready after this many providers for the root CID have - * been found. Providers will continue to be added to the session after this - * until they reach `maxProviders`. - * - * @default 1 - */ - minProviders?: number - - /** - * After this many providers for the root CID have been found, stop searching - * for more providers - * - * @default 3 - */ - maxProviders?: number - +export interface CreateBitswapSessionOptions extends CreateSessionOptions { /** * If true, query connected peers before searching for providers in the * routing @@ -95,14 +79,6 @@ export interface CreateSessionOptions extends AbortOptions, ProgressOptions { }) const session = await bitswap.createSession(cid) - expect(session.peers.size).to.equal(DEFAULT_MIN_PROVIDERS_BEFORE_SESSION_READY) + expect(session.peers.size).to.equal(DEFAULT_SESSION_MIN_PROVIDERS) expect([...session.peers].map(p => p.toString())).to.include(providers[0].id.toString()) // dialed connected peer first @@ -175,12 +175,12 @@ describe('bitswap', () => { // the query continues after the session is ready await delay(100) - // should have continued querying until we reach DEFAULT_MAX_PROVIDERS_PER_REQUEST + // should have continued querying until we reach DEFAULT_SESSION_MAX_PROVIDERS expect(providers[1].id.equals(components.libp2p.dialProtocol.getCall(2).args[0].toString())).to.be.true() expect(providers[2].id.equals(components.libp2p.dialProtocol.getCall(3).args[0].toString())).to.be.true() - // should have stopped at DEFAULT_MAX_PROVIDERS_PER_REQUEST - expect(session.peers.size).to.equal(DEFAULT_MAX_PROVIDERS_PER_REQUEST) + // should have stopped at DEFAULT_SESSION_MAX_PROVIDERS + expect(session.peers.size).to.equal(DEFAULT_SESSION_MAX_PROVIDERS) }) it('should error when creating a session when no peers or providers have the block', async () => { From 9e54ac188ec2eb52a5ff8ef8173abe2d7fb2b238 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 15 Feb 2024 09:10:03 +0000 Subject: [PATCH 17/27] chore: remove extra deps --- packages/bitswap/package.json | 29 +- packages/bitswap/src/bitswap.ts | 382 ++------------- packages/bitswap/src/constants.ts | 2 + packages/bitswap/src/index.ts | 8 - packages/bitswap/src/network.ts | 48 +- packages/bitswap/src/notifications.ts | 137 ------ packages/bitswap/src/peer-want-lists/index.ts | 19 +- .../bitswap/src/peer-want-lists/ledger.ts | 87 ++-- packages/bitswap/src/session.ts | 129 ++++- packages/bitswap/src/stats.ts | 5 +- packages/bitswap/src/want-list.ts | 392 ++++++++++++--- packages/bitswap/test/bitswap.spec.ts | 145 +++--- packages/bitswap/test/network.spec.ts | 63 ++- packages/bitswap/test/notifications.spec.ts | 71 --- packages/bitswap/test/peer-want-list.spec.ts | 449 +++++++++++------- packages/bitswap/test/session.spec.ts | 55 ++- packages/bitswap/test/want-list.spec.ts | 56 +-- 17 files changed, 1039 insertions(+), 1038 deletions(-) delete mode 100644 packages/bitswap/src/notifications.ts delete mode 100644 packages/bitswap/test/notifications.spec.ts diff --git a/packages/bitswap/package.json b/packages/bitswap/package.json index 3e9b1ef4..c47a8f7b 100644 --- a/packages/bitswap/package.json +++ b/packages/bitswap/package.json @@ -148,31 +148,31 @@ "docs": "aegir docs" }, "dependencies": { - "@helia/interface": "^3.0.1", - "@libp2p/interface": "^1.1.1", - "@libp2p/logger": "^4.0.4", - "@libp2p/peer-collections": "^5.1.5", - "@libp2p/utils": "^5.2.0", - "@multiformats/multiaddr": "^12.1.0", + "@helia/interface": "^4.0.0", + "@libp2p/interface": "^1.1.2", + "@libp2p/logger": "^4.0.5", + "@libp2p/peer-collections": "^5.1.6", + "@libp2p/utils": "^5.2.3", + "@multiformats/multiaddr": "^12.1.14", "@multiformats/multiaddr-matcher": "^1.1.2", "any-signal": "^4.1.1", "debug": "^4.3.4", - "events": "^3.3.0", - "interface-blockstore": "^5.2.7", - "interface-store": "^5.1.5", + "interface-blockstore": "^5.2.9", + "interface-store": "^5.1.7", "it-all": "^3.0.4", "it-drain": "^3.0.5", "it-filter": "^3.0.4", "it-length-prefixed": "^9.0.0", "it-length-prefixed-stream": "^1.1.6", "it-map": "^3.0.5", + "it-merge": "^3.0.3", "it-pipe": "^3.0.1", "it-take": "^3.0.1", - "multiformats": "^13.0.0", + "multiformats": "^13.0.1", "p-defer": "^4.0.0", "progress-events": "^1.0.0", "protons-runtime": "^5.0.0", - "race-signal": "^1.0.2", + "race-event": "^1.2.0", "uint8-varint": "^2.0.3", "uint8arraylist": "^2.4.3", "uint8arrays": "^5.0.1" @@ -180,13 +180,8 @@ "devDependencies": { "@libp2p/interface-compliance-tests": "^5.1.3", "@libp2p/peer-id-factory": "^4.0.5", - "@types/lodash.difference": "^4.5.7", - "@types/lodash.flatten": "^4.4.7", - "@types/lodash.range": "^3.2.7", "@types/sinon": "^17.0.3", - "@types/stats-lite": "^2.2.0", - "@types/varint": "^6.0.0", - "aegir": "^42.1.0", + "aegir": "^42.2.2", "blockstore-core": "^4.3.10", "delay": "^6.0.0", "it-pair": "^2.0.6", diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts index 204eb3ac..e9f4dc87 100644 --- a/packages/bitswap/src/bitswap.ts +++ b/packages/bitswap/src/bitswap.ts @@ -1,30 +1,18 @@ /* eslint-disable no-loop-func */ import { DEFAULT_SESSION_MAX_PROVIDERS, DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_QUERY_CONCURRENCY } from '@helia/interface' import { setMaxListeners } from '@libp2p/interface' -import { PeerSet } from '@libp2p/peer-collections' -import { PeerQueue } from '@libp2p/utils/peer-queue' import { anySignal } from 'any-signal' -import drain from 'it-drain' -import { CID } from 'multiformats/cid' -import { sha256 } from 'multiformats/hashes/sha2' -import pDefer from 'p-defer' -import { CodeError } from 'protons-runtime' -import { raceSignal } from 'race-signal' -import { DEFAULT_SESSION_ROOT_PRIORITY } from './constants.js' import { Network } from './network.js' -import { Notifications, receivedBlockEvent, type ReceivedBlockListener, type HaveBlockListener, haveEvent, type DoNotHaveBlockListener, doNotHaveEvent } from './notifications.js' -import { BlockPresenceType, WantType } from './pb/message.js' import { PeerWantLists } from './peer-want-lists/index.js' import { createBitswapSession } from './session.js' import { Stats } from './stats.js' -import vd from './utils/varint-decoder.js' import { WantList } from './want-list.js' -import type { BitswapOptions, Bitswap as BitswapInterface, MultihashHasherLoader, BitswapWantProgressEvents, BitswapNotifyProgressEvents, BitswapSession, WantListEntry, BitswapComponents, CreateBitswapSessionOptions } from './index.js' -import type { BitswapMessage } from './pb/message.js' -import type { ComponentLogger, Libp2p, PeerId } from '@libp2p/interface' +import type { BitswapOptions, Bitswap as BitswapInterface, BitswapWantProgressEvents, BitswapNotifyProgressEvents, BitswapSession, WantListEntry, BitswapComponents, CreateBitswapSessionOptions } from './index.js' +import type { ComponentLogger, PeerId } from '@libp2p/interface' import type { Logger } from '@libp2p/logger' import type { AbortOptions } from '@multiformats/multiaddr' import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' export interface WantOptions extends AbortOptions, ProgressOptions { @@ -49,41 +37,19 @@ export class Bitswap implements BitswapInterface { public blockstore: Blockstore public peerWantLists: PeerWantLists public wantList: WantList - public notifications: Notifications public status: 'starting' | 'started' | 'stopping' | 'stopped' - private readonly hashLoader?: MultihashHasherLoader - - private readonly libp2p: Libp2p constructor (components: BitswapComponents, init: BitswapOptions = {}) { this.logger = components.logger this.log = components.logger.forComponent('helia:bitswap') this.status = 'stopped' - this.libp2p = components.libp2p this.blockstore = components.blockstore - this.hashLoader = init.hashLoader // report stats to libp2p metrics this.stats = new Stats(components) // the network delivers messages this.network = new Network(components, init) - this.network.addEventListener('bitswap:message', evt => { - this._receiveMessage(evt.detail.peer, evt.detail.message) - .catch(err => { - this.log.error('error receiving bitswap message from %p', evt.detail.peer, err) - }) - }) - this.network.addEventListener('peer:connected', evt => { - this.wantList.connected(evt.detail) - .catch(err => { - this.log.error('error processing newly connected bitswap peer %p', evt.detail, err) - }) - }) - this.network.addEventListener('peer:disconnected', evt => { - this.wantList.disconnected(evt.detail) - this.peerWantLists.peerDisconnected(evt.detail) - }) // handle which blocks we send to peers this.peerWantLists = new PeerWantLists({ @@ -96,88 +62,9 @@ export class Bitswap implements BitswapInterface { ...components, network: this.network }) - - // event emitter that lets sessions/want promises know blocks have arrived - this.notifications = new Notifications(components) - } - - /** - * handle messages received through the network - */ - async _receiveMessage (peerId: PeerId, message: BitswapMessage): Promise { - // CIDs we received - const received: CID[] = [] - - // process all incoming blocks - const self = this - await drain(this.blockstore.putMany(async function * () { - for (const block of message.blocks) { - if (block.prefix == null || block.data == null) { - continue - } - - self.log('received block') - const values = vd(block.prefix) - const cidVersion = values[0] - const multicodec = values[1] - const hashAlg = values[2] - // const hashLen = values[3] // We haven't need to use this so far - - const hasher = hashAlg === sha256.code ? sha256 : await self.hashLoader?.getHasher(hashAlg) - - if (hasher == null) { - self.log.error('unknown hash algorithm', hashAlg) - continue - } - - const hash = await hasher.digest(block.data) - const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash) - const wasWanted = self.notifications.listenerCount(receivedBlockEvent(cid)) > 0 - received.push(cid) - - const has = await self.blockstore.has(cid) - - self._updateReceiveCounters(peerId, block.data, has) - - if (!wasWanted) { - continue - } - - if (!has) { - yield { cid, block: block.data } - } - - self.notifications.receivedBlock(cid, block.data, peerId) - } - }())) - - // quickly send out cancels, reduces chances of duplicate block receives - if (received.length > 0) { - this.wantList.cancelWants(received) - .catch(err => { - this.log.error('error sending block cancels', err) - }) - } - - // notify sessions of block haves/don't haves - for (const { cid: cidBytes, type } of message.blockPresences) { - const cid = CID.decode(cidBytes) - - if (type === BlockPresenceType.HaveBlock) { - this.notifications.haveBlock(cid, peerId) - } else { - this.notifications.doNotHaveBlock(cid, peerId) - } - } - - try { - // Respond to any wants in the message - await this.peerWantLists.messageReceived(peerId, message) - } catch (err) { - this.log('failed to receive message from %p', peerId, message) - } } + // TODO: remove me _updateReceiveCounters (peerId: PeerId, data: Uint8Array, exists: boolean): void { this.stats.updateBlocksReceived(1, peerId) this.stats.updateDataReceived(data.byteLength, peerId) @@ -192,257 +79,50 @@ export class Bitswap implements BitswapInterface { const minProviders = options?.minProviders ?? DEFAULT_SESSION_MIN_PROVIDERS const maxProviders = options?.maxProviders ?? DEFAULT_SESSION_MAX_PROVIDERS - // normalize to v1 CID - root = root.toV1() - - const deferred = pDefer() - const session = createBitswapSession({ - notifications: this.notifications, + return createBitswapSession({ wantList: this.wantList, network: this.network, logger: this.logger }, { - root - }) - - let peerDoesNotHave = 0 - let searchedForProviders = false - - const queue = new PeerQueue({ - concurrency: options?.queryConcurrency ?? DEFAULT_SESSION_QUERY_CONCURRENCY - }) - queue.addEventListener('error', (err) => { - this.log.error('error querying peer for %c', root, err) + root, + queryConcurrency: options?.queryConcurrency ?? DEFAULT_SESSION_QUERY_CONCURRENCY, + minProviders, + maxProviders, + connectedPeers: options?.queryConnectedPeers !== false ? [...this.wantList.peers.keys()] : [], + signal: options?.signal }) - queue.addEventListener('completed', () => { - if (session.peers.size === maxProviders) { - this.notifications.removeListener(receivedBlockEvent(root), receivedBlockListener) - this.notifications.removeListener(haveEvent(root), haveBlockListener) - this.notifications.removeListener(doNotHaveEvent(root), doNotHaveBlockListener) - - queue.clear() - } - }) - - const queriedPeers = new PeerSet() - const existingPeers = new PeerSet() - const providerPeers = new PeerSet() - - // register for peer responses - const receivedBlockListener: ReceivedBlockListener = (block, peer): void => { - this.log('adding %p to session after receiving block when asking for HAVE_BLOCK', peer) - session.peers.add(peer) - - // check if the session can be used now - if (session.peers.size === minProviders) { - deferred.resolve(session) - } - - if (session.peers.size === maxProviders) { - this.notifications.removeListener(receivedBlockEvent(root), receivedBlockListener) - this.notifications.removeListener(haveEvent(root), haveBlockListener) - this.notifications.removeListener(doNotHaveEvent(root), doNotHaveBlockListener) - - queue.clear() - } - } - const haveBlockListener: HaveBlockListener = (peer): void => { - this.log('adding %p to session after receiving HAVE_BLOCK', peer) - session.peers.add(peer) - - // check if the session can be used now - if (session.peers.size === minProviders) { - deferred.resolve(session) - } - - if (session.peers.size === maxProviders) { - this.notifications.removeListener(receivedBlockEvent(root), receivedBlockListener) - this.notifications.removeListener(haveEvent(root), haveBlockListener) - this.notifications.removeListener(doNotHaveEvent(root), doNotHaveBlockListener) - - queue.clear() - } - } - const doNotHaveBlockListener: DoNotHaveBlockListener = (peer) => { - peerDoesNotHave++ - - if (searchedForProviders && peerDoesNotHave === queriedPeers.size) { - // no queried peers can supply the root block - deferred.reject(new CodeError(`No peers or providers had ${root}`, 'ERR_NO_PROVIDERS_FOUND')) - } - } - - this.notifications.addListener(receivedBlockEvent(root), receivedBlockListener) - this.notifications.addListener(haveEvent(root), haveBlockListener) - this.notifications.addListener(doNotHaveEvent(root), doNotHaveBlockListener) - - if (options?.queryConnectedPeers !== false) { - // ask our current bitswap peers for the CID - await Promise.all([ - ...this.wantList.peers.keys() - ].map(async (peerId) => { - if (queriedPeers.has(peerId)) { - return - } - - existingPeers.add(peerId) - - await queue.add(async () => { - try { - await this.network.sendMessage(peerId, { - wantlist: { - entries: [{ - cid: root.bytes, - priority: options?.priority ?? DEFAULT_SESSION_ROOT_PRIORITY, - wantType: WantType.WantHave, - sendDontHave: true - }] - } - }, options) - - queriedPeers.add(peerId) - } catch (err: any) { - this.log.error('error querying connected peer %p for initial session', peerId, err) - } - }, { - peerId - }) - })) - - this.log.trace('creating session queried %d connected peers for %c', queriedPeers, root) - } - - // find network providers too but do not wait for the query to complete - void Promise.resolve().then(async () => { - if (options?.queryRoutingPeers === false) { - return - } - - let providers = 0 - - for await (const provider of this.network.findProviders(root, options)) { - providers++ - - if (queriedPeers.has(provider.id)) { - continue - } - - await queue.add(async () => { - try { - await this.network.sendMessage(provider.id, { - wantlist: { - entries: [{ - cid: root.bytes, - priority: options?.priority ?? DEFAULT_SESSION_ROOT_PRIORITY, - wantType: WantType.WantHave, - sendDontHave: true - }] - } - }, options) - - providerPeers.add(provider.id) - queriedPeers.add(provider.id) - } catch (err: any) { - this.log.error('error querying provider %p for initial session', provider.id, err.errors ?? err) - } - }, { - peerId: provider.id - }) - - if (session.peers.size === maxProviders) { - break - } - } - - this.log.trace('creating session found %d providers for %c', providers, root) - - searchedForProviders = true - - // we have no peers and could find no providers - if (providers === 0) { - deferred.reject(new CodeError(`Could not find providers for ${root}`, 'ERR_NO_PROVIDERS_FOUND')) - } - }) - .catch(err => { - this.log.error('error querying providers for %c', root, err) - }) - .finally(() => { - if (peerDoesNotHave === queriedPeers.size) { - // no queried peers can supply the root block - deferred.reject(new CodeError(`No peers or providers had ${root}`, 'ERR_NO_PROVIDERS_FOUND')) - } - }) - - return raceSignal(deferred.promise, options?.signal) } async want (cid: CID, options: WantOptions = {}): Promise { - const loadOrFetchFromNetwork = async (cid: CID, wantBlockPromise: Promise, options: WantOptions): Promise => { - try { - // have to await here as we want to handle ERR_NOT_FOUND from the - // blockstore - return await Promise.race([ - this.blockstore.get(cid, options), - wantBlockPromise - ]) - } catch (err: any) { - if (err.code !== 'ERR_NOT_FOUND') { - throw err - } - - // add the block to the wantlist - this.wantList.wantBlocks([cid]) - .catch(err => { - this.log.error('error adding %c to wantlist', cid, err) - }) - - // find providers and connect to them - this.network.findAndConnect(cid, options) - .catch(err => { - this.log.error('could not find and connect for cid %c', cid, err) - }) - - return wantBlockPromise - } - } - - // it's possible for blocks to come in while we do the async operations to - // get them from the blockstore leading to a race condition, so register for - // incoming block notifications as well as trying to get it from the - // datastore const controller = new AbortController() setMaxListeners(Infinity, controller.signal) const signal = anySignal([controller.signal, options.signal]) + // find providers and connect to them + this.network.findAndConnect(cid, { + ...options, + signal + }) + .catch(err => { + // if the controller was aborted we found the block already so ignore + // the error + if (!controller.signal.aborted) { + this.log.error('error during finding and connect for cid %c', cid, err) + } + }) + try { - const wantBlockPromise = this.notifications.wantBlock(cid, { + const result = await this.wantList.wantBlock(cid, { ...options, signal }) - const block = await Promise.race([ - wantBlockPromise, - loadOrFetchFromNetwork(cid, wantBlockPromise, { - ...options, - signal - }) - ]) - - return block - } catch (err: any) { - if (err.code === 'ERR_ABORTED') { - // the want was cancelled, send out cancel messages - await this.wantList.cancelWants([cid]) - } - - throw err + return result.block } finally { // since we have the block we can now abort any outstanding attempts to - // fetch it + // find providers for it controller.abort() signal.clear() - - this.wantList.unwantBlocks([cid]) } } @@ -450,13 +130,17 @@ export class Bitswap implements BitswapInterface { * Sends notifications about the arrival of a block */ async notify (cid: CID, block: Uint8Array, options: ProgressOptions & AbortOptions = {}): Promise { - this.notifications.receivedBlock(cid, block, this.libp2p.peerId) - await this.peerWantLists.receivedBlock(cid, options) } getWantlist (): WantListEntry[] { return [...this.wantList.wants.values()] + .filter(entry => !entry.cancel) + .map(entry => ({ + cid: entry.cid, + priority: entry.priority, + wantType: entry.wantType + })) } getPeerWantlist (peer: PeerId): WantListEntry[] | undefined { diff --git a/packages/bitswap/src/constants.ts b/packages/bitswap/src/constants.ts index 3671f037..8657799b 100644 --- a/packages/bitswap/src/constants.ts +++ b/packages/bitswap/src/constants.ts @@ -3,7 +3,9 @@ export const DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK = 1024 export const DEFAULT_MAX_INBOUND_STREAMS = 1024 export const DEFAULT_MAX_OUTBOUND_STREAMS = 1024 export const DEFAULT_MESSAGE_RECEIVE_TIMEOUT = 5000 +export const DEFAULT_MESSAGE_SEND_DELAY = 10 export const DEFAULT_MESSAGE_SEND_TIMEOUT = 5000 export const DEFAULT_MESSAGE_SEND_CONCURRENCY = 50 export const DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS = false export const DEFAULT_SESSION_ROOT_PRIORITY = 1 +export const DEFAULT_MAX_PROVIDERS_PER_REQUEST = 3 diff --git a/packages/bitswap/src/index.ts b/packages/bitswap/src/index.ts index 9c441a8b..b7641b93 100644 --- a/packages/bitswap/src/index.ts +++ b/packages/bitswap/src/index.ts @@ -45,16 +45,8 @@ export interface BitswapSession { export interface WantListEntry { cid: CID - session: PeerSet priority: number wantType: WantType - cancel: boolean - sendDontHave: boolean - - /** - * Whether we have sent the dont-have block presence - */ - sentDontHave?: boolean } export interface CreateBitswapSessionOptions extends CreateSessionOptions { diff --git a/packages/bitswap/src/network.ts b/packages/bitswap/src/network.ts index c4a53e4e..17dc36fb 100644 --- a/packages/bitswap/src/network.ts +++ b/packages/bitswap/src/network.ts @@ -12,6 +12,7 @@ import take from 'it-take' import { base64 } from 'multiformats/bases/base64' import { CID } from 'multiformats/cid' import { CustomProgressEvent } from 'progress-events' +import { raceEvent } from 'race-event' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_TIMEOUT, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js' import { BitswapMessage } from './pb/message.js' @@ -19,7 +20,7 @@ import type { WantOptions } from './bitswap.js' import type { MultihashHasherLoader } from './index.js' import type { Block, BlockPresence, WantlistEntry } from './pb/message.js' import type { Provider, Routing } from '@helia/interface/routing' -import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, MetricGroup, ComponentLogger, Metrics } from '@libp2p/interface' +import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, MetricGroup, ComponentLogger, Metrics, IdentifyResult } from '@libp2p/interface' import type { Logger } from '@libp2p/logger' import type { ProgressEvent, ProgressOptions } from 'progress-events' @@ -81,6 +82,11 @@ export interface NetworkComponents { metrics?: Metrics } +export interface BitswapMessageEventDetail { + peer: PeerId + message: BitswapMessage +} + export interface NetworkEvents { 'bitswap:message': CustomEvent<{ peer: PeerId, message: BitswapMessage }> 'peer:connected': CustomEvent @@ -255,7 +261,7 @@ export class Network extends TypedEventEmitter { } /** - * Find providers given a `cid`. + * Find bitswap providers for a given `cid`. */ async * findProviders (cid: CID, options?: AbortOptions & ProgressOptions): AsyncIterable { options?.onProgress?.(new CustomProgressEvent('bitswap:network:find-providers', cid)) @@ -267,7 +273,7 @@ export class Network extends TypedEventEmitter { let hasDirectAddress = false for (let ma of provider.multiaddrs) { - if (ma.getPeerId() === null) { + if (ma.getPeerId() == null) { ma = ma.encapsulate(`/p2p/${provider.id}`) } @@ -282,6 +288,11 @@ export class Network extends TypedEventEmitter { } } + // ignore non-bitswap providers + if (provider.protocols?.includes('transport-bitswap') === false) { + continue + } + yield provider } } @@ -292,11 +303,7 @@ export class Network extends TypedEventEmitter { async findAndConnect (cid: CID, options?: WantOptions): Promise { await drain( take( - map(this.findProviders(cid, options), async provider => this.connectTo(provider.id, options) - .catch(err => { - // Prevent unhandled promise rejection - this.log.error(err) - })), + map(this.findProviders(cid, options), async provider => this.connectTo(provider.id, options)), options?.maxProviders ?? DEFAULT_MAX_PROVIDERS_PER_REQUEST ) ) @@ -384,7 +391,30 @@ export class Network extends TypedEventEmitter { } options?.onProgress?.(new CustomProgressEvent('bitswap:network:dial', peer)) - return this.libp2p.dial(peer, options) + + // dial and wait for identify - this is to avoid opening a protocol stream + // that we are not going to use but depends on the remote node running the + // identitfy protocol + const [ + connection + ] = await Promise.all([ + this.libp2p.dial(peer, options), + raceEvent(this.libp2p, 'peer:identify', options?.signal, { + filter: (evt: CustomEvent): boolean => { + if (!evt.detail.peerId.equals(peer)) { + return false + } + + if (evt.detail.protocols.includes(BITSWAP_120)) { + return true + } + + throw new CodeError(`${peer} did not support ${BITSWAP_120}`, 'ERR_BITSWAP_UNSUPPORTED_BY_PEER') + } + }) + ]) + + return connection } _updateSentStats (peerId: PeerId, blocks: Block[] = []): void { diff --git a/packages/bitswap/src/notifications.ts b/packages/bitswap/src/notifications.ts deleted file mode 100644 index c7075c87..00000000 --- a/packages/bitswap/src/notifications.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { EventEmitter } from 'events' -import { CodeError } from '@libp2p/interface' -import { CustomProgressEvent, type ProgressOptions } from 'progress-events' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { BitswapWantBlockProgressEvents } from './index.js' -import type { AbortOptions, ComponentLogger, PeerId } from '@libp2p/interface' -import type { Logger } from '@libp2p/logger' -import type { CID } from 'multiformats/cid' - -/** - * Return the event name for an unwant of the passed CID - */ -export const unwantEvent = (cid: CID): string => `unwant:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` - -/** - * Return the event name for the receipt of the block for the passed CID - */ -export const receivedBlockEvent = (cid: CID): string => `block:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` - -/** - * Return the event name for a peer telling us they have the block for the - * passed CID - */ -export const haveEvent = (cid: CID): string => `have:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` - -/** - * Return the event name for a peer telling us they do not have the block for - * the passed CID - */ -export const doNotHaveEvent = (cid: CID): string => `do-not-have:${uint8ArrayToString(cid.multihash.bytes, 'base64')}` - -export interface ReceivedBlockListener { - (block: Uint8Array, peer: PeerId): void -} - -export interface HaveBlockListener { - (peer: PeerId): void -} - -export interface DoNotHaveBlockListener { - (peer: PeerId): void -} - -export interface NotificationsComponents { - logger: ComponentLogger -} - -export class Notifications extends EventEmitter { - private readonly log: Logger - - /** - * Internal module used to track events about incoming blocks, - * wants and unwants. - */ - constructor (components: NotificationsComponents) { - super() - - this.setMaxListeners(Infinity) - this.log = components.logger.forComponent('helia:bitswap:notifications') - } - - /** - * Signal the system that the passed peer has the block - */ - haveBlock (cid: CID, peer: PeerId): void { - const event = haveEvent(cid) - this.log(event) - this.emit(event, peer) - } - - /** - * Signal the system that the passed peer does not have block - */ - doNotHaveBlock (cid: CID, peer: PeerId): void { - const event = doNotHaveEvent(cid) - this.log(event) - this.emit(event, peer) - } - - /** - * Signal the system that we received `block` from the passed peer - */ - receivedBlock (cid: CID, block: Uint8Array, peer: PeerId): void { - const event = receivedBlockEvent(cid) - this.log(event) - this.emit(event, block, peer) - } - - /** - * Signal the system that we are waiting to receive the block associated with - * the given `cid`. - * - * Returns a Promise that resolves to the block when it is received, or - * rejects if the block is unwanted. - */ - async wantBlock (cid: CID, options: AbortOptions & ProgressOptions = {}): Promise { - const blockEvt = receivedBlockEvent(cid) - const unwantEvt = unwantEvent(cid) - - this.log(`wantBlock:${cid}`) - - return new Promise((resolve, reject) => { - const onUnwant = (): void => { - this.removeListener(blockEvt, onBlock) - - options.onProgress?.(new CustomProgressEvent('bitswap:want-block:unwant', cid)) - reject(new CodeError(`Block for ${cid} unwanted`, 'ERR_UNWANTED')) - } - - const onBlock = (data: Uint8Array): void => { - this.removeListener(unwantEvt, onUnwant) - - options.onProgress?.(new CustomProgressEvent('bitswap:want-block:block', cid)) - resolve(data) - } - - this.once(unwantEvt, onUnwant) - this.once(blockEvt, onBlock) - - options.signal?.addEventListener('abort', () => { - this.removeListener(blockEvt, onBlock) - this.removeListener(unwantEvt, onUnwant) - - reject(new CodeError(`Want for ${cid} aborted`, 'ERR_ABORTED')) - }) - }) - } - - /** - * Signal that the block is not wanted any more - */ - unwantBlock (cid: CID): void { - const event = unwantEvent(cid) - this.log(event) - this.emit(event) - } -} diff --git a/packages/bitswap/src/peer-want-lists/index.ts b/packages/bitswap/src/peer-want-lists/index.ts index 692c4d91..bcd3b8fc 100644 --- a/packages/bitswap/src/peer-want-lists/index.ts +++ b/packages/bitswap/src/peer-want-lists/index.ts @@ -1,4 +1,4 @@ -import { trackedPeerMap, PeerSet } from '@libp2p/peer-collections' +import { trackedPeerMap } from '@libp2p/peer-collections' import { CID } from 'multiformats/cid' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { WantType } from '../pb/message.js' @@ -48,6 +48,16 @@ export class PeerWantLists { name: 'ipfs_bitswap_ledger_map', metrics: components.metrics }) + + this.network.addEventListener('bitswap:message', (evt) => { + this.receiveMessage(evt.detail.peer, evt.detail.message) + .catch(err => { + this.log.error('error receiving bitswap message from %p', evt.detail.peer, err) + }) + }) + this.network.addEventListener('peer:disconnected', evt => { + this.peerDisconnected(evt.detail) + }) } ledgerForPeer (peerId: PeerId): PeerLedger | undefined { @@ -83,7 +93,7 @@ export class PeerWantLists { /** * Handle incoming messages */ - async messageReceived (peerId: PeerId, message: BitswapMessage): Promise { + async receiveMessage (peerId: PeerId, message: BitswapMessage): Promise { let ledger = this.ledgerMap.get(peerId) if (ledger == null) { @@ -113,7 +123,6 @@ export class PeerWantLists { if (entry.cancel === true) { this.log('peer %p cancelled want of block for %c', peerId, cid) - ledger.wants.delete(cidStr) } else { if (entry.wantType === WantType.WantHave) { @@ -124,11 +133,9 @@ export class PeerWantLists { ledger.wants.set(cidStr, { cid, - session: new PeerSet(), priority: entry.priority, wantType: entry.wantType ?? WantType.WantBlock, - sendDontHave: entry.sendDontHave ?? false, - cancel: entry.cancel ?? false + sendDontHave: entry.sendDontHave ?? false }) } } diff --git a/packages/bitswap/src/peer-want-lists/ledger.ts b/packages/bitswap/src/peer-want-lists/ledger.ts index 314556da..11d849ca 100644 --- a/packages/bitswap/src/peer-want-lists/ledger.ts +++ b/packages/bitswap/src/peer-want-lists/ledger.ts @@ -2,11 +2,11 @@ import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../constants.js' import { BlockPresenceType, type BitswapMessage, WantType } from '../pb/message.js' import { cidToPrefix } from '../utils/cid-prefix.js' -import type { WantListEntry } from '../index.js' import type { Network } from '../network.js' import type { PeerId } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' import type { AbortOptions } from 'it-length-prefixed-stream' +import type { CID } from 'multiformats/cid' export interface LedgerComponents { peerId: PeerId @@ -18,11 +18,39 @@ export interface LedgerInit { maxSizeReplaceHasWithBlock?: number } +export interface PeerWantListEntry { + /** + * The CID the peer has requested + */ + cid: CID + + /** + * The priority with which the remote should return the block + */ + priority: number + + /** + * If we want the block or if we want the remote to tell us if they have the + * block - note if the block is small they'll send it to us anyway. + */ + wantType: WantType + + /** + * Whether the remote should tell us if they have the block or not + */ + sendDontHave: boolean + + /** + * If we don't have the block and we've told them we don't have the block + */ + sentDontHave?: boolean +} + export class Ledger { public peerId: PeerId private readonly blockstore: Blockstore private readonly network: Network - public wants: Map + public wants: Map public exchangeCount: number public bytesSent: number public bytesReceived: number @@ -65,17 +93,7 @@ export class Ledger { const sentBlocks = new Set() for (const [key, entry] of this.wants.entries()) { - let block: Uint8Array | undefined - let has = false - - try { - block = await this.blockstore.get(entry.cid, options) - has = true - } catch (err: any) { - if (err.code !== 'ERR_NOT_FOUND') { - throw err - } - } + const has = await this.blockstore.has(entry.cid, options) if (!has) { // we don't have the requested block and the remote is not interested @@ -84,6 +102,11 @@ export class Ledger { continue } + // we have already told them we don't have the block + if (entry.sentDontHave === true) { + continue + } + entry.sentDontHave = true message.blockPresences.push({ cid: entry.cid.bytes, @@ -93,31 +116,31 @@ export class Ledger { continue } - if (block != null) { - // have the requested block - if (entry.wantType === WantType.WantHave) { - if (block.byteLength < this.maxSizeReplaceHasWithBlock) { - // send it anyway - sentBlocks.add(key) - message.blocks.push({ - data: block, - prefix: cidToPrefix(entry.cid) - }) - } else { - // tell them we have the block - message.blockPresences.push({ - cid: entry.cid.bytes, - type: BlockPresenceType.HaveBlock - }) - } - } else { - // they want the block, send it to them + const block = await this.blockstore.get(entry.cid, options) + + // do they want the block or just us to tell them we have the block + if (entry.wantType === WantType.WantHave) { + if (block.byteLength < this.maxSizeReplaceHasWithBlock) { + // if the block is small we just send it to them sentBlocks.add(key) message.blocks.push({ data: block, prefix: cidToPrefix(entry.cid) }) + } else { + // otherwise tell them we have the block + message.blockPresences.push({ + cid: entry.cid.bytes, + type: BlockPresenceType.HaveBlock + }) } + } else { + // they want the block, send it to them + sentBlocks.add(key) + message.blocks.push({ + data: block, + prefix: cidToPrefix(entry.cid) + }) } } diff --git a/packages/bitswap/src/session.ts b/packages/bitswap/src/session.ts index 7d0c8f2b..d9f7c691 100644 --- a/packages/bitswap/src/session.ts +++ b/packages/bitswap/src/session.ts @@ -1,67 +1,150 @@ import { CodeError } from '@libp2p/interface' import { PeerSet } from '@libp2p/peer-collections' +import { PeerQueue } from '@libp2p/utils/peer-queue' +import map from 'it-map' +import merge from 'it-merge' +import pDefer, { type DeferredPromise } from 'p-defer' import type { BitswapWantProgressEvents, BitswapSession as BitswapSessionInterface } from './index.js' import type { Network } from './network.js' -import type { Notifications } from './notifications.js' import type { WantList } from './want-list.js' -import type { ComponentLogger, Logger } from '@libp2p/interface' +import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface' import type { AbortOptions } from 'interface-store' import type { CID } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' export interface BitswapSessionComponents { - notifications: Notifications network: Network wantList: WantList logger: ComponentLogger } -export interface BitswapSessionInit { +export interface BitswapSessionInit extends AbortOptions { root: CID + queryConcurrency: number + minProviders: number + maxProviders: number + connectedPeers: PeerId[] } class BitswapSession implements BitswapSessionInterface { public readonly root: CID public readonly peers: PeerSet private readonly log: Logger - private readonly notifications: Notifications private readonly wantList: WantList + private readonly network: Network + private readonly queue: PeerQueue + private readonly maxProviders: number constructor (components: BitswapSessionComponents, init: BitswapSessionInit) { this.peers = new PeerSet() this.root = init.root - this.log = components.logger.forComponent(`bitswap:session:${init.root}`) - this.notifications = components.notifications + this.maxProviders = init.maxProviders + this.log = components.logger.forComponent(`helia:bitswap:session:${init.root}`) this.wantList = components.wantList + this.network = components.network + + this.queue = new PeerQueue({ + concurrency: init.queryConcurrency + }) + this.queue.addEventListener('error', (evt) => { + this.log.error('error querying peer for %c', this.root, evt.detail) + }) } - async want (cid: CID, options?: AbortOptions & ProgressOptions): Promise { + async want (cid: CID, options: AbortOptions & ProgressOptions = {}): Promise { if (this.peers.size === 0) { throw new CodeError('Bitswap session had no peers', 'ERR_NO_SESSION_PEERS') } - // normalize to v1 CID - cid = cid.toV1() - this.log('sending WANT-BLOCK for %c to', cid, this.peers) - await this.wantList.wantBlocks([cid], { - session: this.peers, - sendDontHave: true - }) + const result = await Promise.any( + [...this.peers].map(async peerId => { + return this.wantList.wantBlock(cid, { + peerId, + ...options + }) + }) + ) - const block = await this.notifications.wantBlock(cid, options) + this.log('received block for %c from %p', cid, result.sender) - this.log('sending cancels for %c to', cid, this.peers) + // TODO findNewProviders when promise.any throws aggregate error and signal + // is not aborted - await this.wantList.cancelWants([cid], { - session: this.peers - }) + return result.block + } + + async findNewProviders (cid: CID, count: number, options: AbortOptions = {}): Promise { + const deferred: DeferredPromise = pDefer() + let found = 0 + + this.log('find %d-%d new provider(s) for %c', count, this.maxProviders, cid) + + const source = merge( + [...this.wantList.peers.keys()], + map(this.network.findProviders(cid, options), prov => prov.id) + ) + + void Promise.resolve() + .then(async () => { + for await (const peerId of source) { + // eslint-disable-next-line no-loop-func + await this.queue.add(async () => { + try { + this.log('asking potential session peer %p if they have %c', peerId, cid) + const result = await this.wantList.wantPresence(cid, { + peerId, + ...options + }) - return block + if (!result.has) { + this.log('potential session peer %p did not have %c', peerId, cid) + return + } + + this.log('potential session peer %p had %c', peerId, cid) + found++ + + // add to list + this.peers.add(peerId) + + if (found === count) { + this.log('found %d session peers', found) + + deferred.resolve() + } + + if (found === this.maxProviders) { + this.log('found max provider session peers', found) + + this.queue.clear() + } + } catch (err: any) { + this.log.error('error querying potential session peer %p for %c', peerId, cid, err.errors ?? err) + } + }, { + peerId + }) + } + + this.log('found %d session peers total', found) + + if (count > 0) { + deferred.reject(new CodeError(`Found ${found} of ${count} providers`, 'ERR_NO_PROVIDERS_FOUND')) + } + }) + + return deferred.promise } } -export function createBitswapSession (components: BitswapSessionComponents, init: BitswapSessionInit): BitswapSessionInterface { - return new BitswapSession(components, init) +export async function createBitswapSession (components: BitswapSessionComponents, init: BitswapSessionInit): Promise { + const session = new BitswapSession(components, init) + + await session.findNewProviders(init.root, init.minProviders, { + signal: init.signal + }) + + return session } diff --git a/packages/bitswap/src/stats.ts b/packages/bitswap/src/stats.ts index b95bb5a4..9853dab0 100644 --- a/packages/bitswap/src/stats.ts +++ b/packages/bitswap/src/stats.ts @@ -1,19 +1,16 @@ -import { EventEmitter } from 'events' import type { MetricGroup, Metrics, PeerId } from '@libp2p/interface' export interface StatsComponents { metrics?: Metrics } -export class Stats extends EventEmitter { +export class Stats { private readonly blocksReceived?: MetricGroup private readonly duplicateBlocksReceived?: MetricGroup private readonly dataReceived?: MetricGroup private readonly duplicateDataReceived?: MetricGroup constructor (components: StatsComponents) { - super() - this.blocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_received_blocks') this.duplicateBlocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_received_blocks') this.dataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_data_received_bytes') diff --git a/packages/bitswap/src/want-list.ts b/packages/bitswap/src/want-list.ts index 88b4d703..5cd3d632 100644 --- a/packages/bitswap/src/want-list.ts +++ b/packages/bitswap/src/want-list.ts @@ -1,18 +1,25 @@ +import { AbortError } from '@libp2p/interface' import { trackedPeerMap, PeerSet } from '@libp2p/peer-collections' import { trackedMap } from '@libp2p/utils/tracked-map' import all from 'it-all' import filter from 'it-filter' import map from 'it-map' import { pipe } from 'it-pipe' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' +import pDefer from 'p-defer' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { WantType } from './pb/message.js' -import type { WantListEntry } from './index.js' -import type { Network } from './network.js' +import { DEFAULT_MESSAGE_SEND_DELAY } from './constants.js' +import { BlockPresenceType, WantType } from './pb/message.js' +import vd from './utils/varint-decoder.js' +import type { MultihashHasherLoader } from './index.js' +import type { BitswapNetworkWantProgressEvents, Network } from './network.js' import type { BitswapMessage } from './pb/message.js' -import type { ComponentLogger, Metrics, PeerId, Startable } from '@libp2p/interface' +import type { ComponentLogger, Metrics, PeerId, Startable, AbortOptions } from '@libp2p/interface' import type { Logger } from '@libp2p/logger' import type { PeerMap } from '@libp2p/peer-collections' -import type { CID } from 'multiformats/cid' +import type { DeferredPromise } from 'p-defer' +import type { ProgressOptions } from 'progress-events' export interface WantListComponents { network: Network @@ -20,35 +27,87 @@ export interface WantListComponents { metrics?: Metrics } -export interface WantBlocksOptions { +export interface WantListInit { + sendMessagesDelay?: number + hashLoader?: MultihashHasherLoader +} + +export interface WantListEntry { /** - * If set, this wantlist entry will only be sent to peers in the peer set + * The CID we send to the remote */ - session?: PeerSet + cid: CID /** - * Allow prioritsing blocks + * The priority with which the remote should return the block */ - priority?: number + priority: number + + /** + * If we want the block or if we want the remote to tell us if they have the + * block - note if the block is small they'll send it to us anyway. + */ + wantType: WantType + + /** + * Whether we are cancelling the block want or not + */ + cancel: boolean + + /** + * Whether the remote should tell us if they have the block or not + */ + sendDontHave: boolean + + /** + * If this set has members, the want will only be sent to these peers + */ + session: PeerSet /** - * Specify if the remote should send us the block or just tell us they have - * the block + * Promises returned from `.wantBlock` for this block */ - wantType?: WantType + blockWantListeners: Array> /** - * Pass true to get the remote to tell us if they don't have the block rather - * than not replying at all + * Promises returned from `.wantPresence` for this block */ - sendDontHave?: boolean + blockPresenceListeners: Array> +} +export interface WantOptions extends AbortOptions, ProgressOptions { /** - * Pass true to cancel wants with peers + * If set, this WantList entry will only be sent to this peer */ - cancel?: boolean + peerId?: PeerId + + /** + * Allow prioritising blocks + */ + priority?: number +} + +export interface WantBlockResult { + sender: PeerId + cid: CID + block: Uint8Array +} + +export interface WantDontHaveResult { + sender: PeerId + cid: CID + has: false +} + +export interface WantHaveResult { + sender: PeerId + cid: CID + has: true + block?: Uint8Array } +export type WantPresenceResult = WantDontHaveResult | WantHaveResult + export class WantList implements Startable { /** * Tracks what CIDs we've previously sent to which peers @@ -57,8 +116,11 @@ export class WantList implements Startable { public readonly wants: Map private readonly network: Network private readonly log: Logger + private readonly sendMessagesDelay: number + private sendMessagesTimeout?: ReturnType + private readonly hashLoader?: MultihashHasherLoader - constructor (components: WantListComponents) { + constructor (components: WantListComponents, init: WantListInit = {}) { this.peers = trackedPeerMap({ name: 'ipfs_bitswap_peers', metrics: components.metrics @@ -68,54 +130,123 @@ export class WantList implements Startable { metrics: components.metrics }) this.network = components.network - this.log = components.logger.forComponent('helia:bitswap:wantlist:self') + this.sendMessagesDelay = init.sendMessagesDelay ?? DEFAULT_MESSAGE_SEND_DELAY + this.log = components.logger.forComponent('helia:bitswap:wantlist') + this.hashLoader = init.hashLoader + + this.network.addEventListener('bitswap:message', (evt) => { + this.receiveMessage(evt.detail.peer, evt.detail.message) + .catch(err => { + this.log.error('error receiving bitswap message from %p', evt.detail.peer, err) + }) + }) + this.network.addEventListener('peer:connected', evt => { + this.peerConnected(evt.detail) + .catch(err => { + this.log.error('error processing newly connected bitswap peer %p', evt.detail, err) + }) + }) + this.network.addEventListener('peer:disconnected', evt => { + this.peerDisconnected(evt.detail) + }) } - async _addEntries (cids: CID[], options: WantBlocksOptions = {}): Promise { - for (const cid of cids) { - const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') - let entry = this.wants.get(cidStr) - - if (entry == null) { - // we are cancelling a want that's not in our wantlist - if (options.cancel === true) { - continue - } + private async addEntry (cid: CID, options: WantOptions & { wantType: WantType.WantBlock }): Promise + private async addEntry (cid: CID, options: WantOptions & { wantType: WantType.WantHave }): Promise + private async addEntry (cid: CID, options: WantOptions & { wantType: WantType }): Promise { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + let entry = this.wants.get(cidStr) + + if (entry == null) { + entry = { + cid, + session: new PeerSet(), + priority: options.priority ?? 1, + wantType: options.wantType ?? WantType.WantBlock, + cancel: false, + sendDontHave: true, + blockWantListeners: [], + blockPresenceListeners: [] + } - entry = { - cid, - session: options.session ?? new PeerSet(), - priority: options.priority ?? 1, - wantType: options.wantType ?? WantType.WantBlock, - cancel: Boolean(options.cancel), - sendDontHave: Boolean(options.sendDontHave) - } + if (options.peerId != null) { + entry.session.add(options.peerId) } - // upgrade want-have to want-block - if (entry.wantType === WantType.WantHave && options.wantType === WantType.WantBlock) { - entry.wantType = WantType.WantBlock + this.wants.set(cidStr, entry) + } + + // upgrade want-have to want-block if the new want is a WantBlock but the + // previous want was a WantHave + if (entry.wantType === WantType.WantHave && options.wantType === WantType.WantBlock) { + entry.wantType = WantType.WantBlock + } + + // if this want was part of a session.. + if (entry.session.size > 0) { + // if the new want is also part of a session, expand the want session to + // include both sets of peers + if (options.peerId != null) { + entry.session.add(options.peerId) } - // cancel the want if requested to do so - if (options.cancel === true) { - entry.cancel = true + // if the new want is not part of a session, make this want a non-session + // want - nb. this will cause this WantList entry to be sent to every peer + // instead of just the ones in the session + if (options.peerId == null) { + entry.session.clear() } + } + + // add a promise that will be resolved or rejected when the response arrives + let deferred: DeferredPromise + + if (options.wantType === WantType.WantBlock) { + const p = deferred = pDefer() - // if this entry has previously been part of a session but the new want - // is not, make this want a non-session want - if (options.session == null) { - entry.session = new PeerSet() + entry.blockWantListeners.push(p) + } else { + const p = deferred = pDefer() + + entry.blockPresenceListeners.push(p) + } + + // reject the promise if the want is rejected + const abortListener = (): void => { + this.log('want for %c was aborted, cancelling want', cid) + + if (entry != null) { + entry.cancel = true } - this.wants.set(cidStr, entry) + deferred.reject(new AbortError('Want was aborted')) } + options.signal?.addEventListener('abort', abortListener) // broadcast changes - await this.sendMessages() + clearTimeout(this.sendMessagesTimeout) + this.sendMessagesTimeout = setTimeout(() => { + void this.sendMessages() + .catch(err => { + this.log('error sending messages to peers', err) + }) + }, this.sendMessagesDelay) + + try { + return await deferred.promise + } finally { + // remove listener + options.signal?.removeEventListener('abort', abortListener) + // remove deferred promise + if (options.wantType === WantType.WantBlock) { + entry.blockWantListeners = entry.blockWantListeners.filter(recipient => recipient !== deferred) + } else { + entry.blockPresenceListeners = entry.blockPresenceListeners.filter(recipient => recipient !== deferred) + } + } } - async sendMessages (): Promise { + private async sendMessages (): Promise { for (const [peerId, sentWants] of this.peers) { const sent = new Set() const message: Partial = { @@ -185,36 +316,159 @@ export class WantList implements Startable { } } - /** - * Add all the CIDs to the wantlist - */ - async wantBlocks (cids: CID[], options: WantBlocksOptions = {}): Promise { - await this._addEntries(cids, options) + has (cid: CID): boolean { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + return this.wants.has(cidStr) } /** - * Remove CIDs from the wantlist without sending cancel messages + * Add a CID to the wantlist */ - unwantBlocks (cids: CID[], options: WantBlocksOptions = {}): void { - cids.forEach(cid => { + async wantPresence (cid: CID, options: WantOptions = {}): Promise { + if (options.peerId != null && this.peers.get(options.peerId) == null) { const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') - this.wants.delete(cidStr) + try { + // if we don't have them as a peer, add them + this.peers.set(options.peerId, new Set([cidStr])) + + // sending WantHave directly to peer + await this.network.sendMessage(options.peerId, { + wantlist: { + full: false, + entries: [{ + cid: cid.bytes, + sendDontHave: true, + wantType: WantType.WantHave, + priority: 1 + }] + } + }) + } catch (err) { + // sending failed, remove them as a peer + this.peers.delete(options.peerId) + + throw err + } + } + + return this.addEntry(cid, { + ...options, + wantType: WantType.WantHave }) } /** - * Send cancel messages to peers for the passed CIDs + * Add a CID to the wantlist */ - async cancelWants (cids: CID[], options: WantBlocksOptions = {}): Promise { - this.log('cancel wants: %s', cids.length) - await this._addEntries(cids, { + async wantBlock (cid: CID, options: WantOptions = {}): Promise { + return this.addEntry(cid, { ...options, - cancel: true + wantType: WantType.WantBlock }) } - async connected (peerId: PeerId): Promise { + /** + * Invoked when a message is received from a bitswap peer + */ + private async receiveMessage (sender: PeerId, message: BitswapMessage): Promise { + this.log('received message from %p', sender) + + // blocks received + const blockResults: WantBlockResult[] = [] + const presenceResults: WantPresenceResult[] = [] + + // process blocks + for (const block of message.blocks) { + if (block.prefix == null || block.data == null) { + continue + } + + this.log('received block') + const values = vd(block.prefix) + const cidVersion = values[0] + const multicodec = values[1] + const hashAlg = values[2] + // const hashLen = values[3] // We haven't need to use this so far + + const hasher = hashAlg === sha256.code ? sha256 : await this.hashLoader?.getHasher(hashAlg) + + if (hasher == null) { + this.log.error('unknown hash algorithm', hashAlg) + continue + } + + const hash = await hasher.digest(block.data) + const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash) + + this.log('received block from %p for %c', sender, cid) + + blockResults.push({ + sender, + cid, + block: block.data + }) + + presenceResults.push({ + sender, + cid, + has: true + }) + } + + // process block presences + for (const { cid: cidBytes, type } of message.blockPresences) { + const cid = CID.decode(cidBytes) + + this.log('received %s from %p for %c', type, sender, cid) + + presenceResults.push({ + sender, + cid, + has: type === BlockPresenceType.HaveBlock + }) + } + + for (const result of blockResults) { + const cidStr = uint8ArrayToString(result.cid.multihash.bytes, 'base64') + const entry = this.wants.get(cidStr) + + if (entry == null) { + return + } + + const recipients = entry.blockWantListeners + entry.blockWantListeners = [] + recipients.forEach((p) => { + p.resolve(result) + }) + + // since we received the block, flip the cancel flag to send cancels to + // any peers on the next message sending iteration, this will remove it + // from the internal want list + entry.cancel = true + } + + for (const result of presenceResults) { + const cidStr = uint8ArrayToString(result.cid.multihash.bytes, 'base64') + const entry = this.wants.get(cidStr) + + if (entry == null) { + return + } + + const recipients = entry.blockPresenceListeners + entry.blockPresenceListeners = [] + recipients.forEach((p) => { + p.resolve(result) + }) + } + } + + /** + * Invoked when the network topology notices a new peer that supports Bitswap + */ + async peerConnected (peerId: PeerId): Promise { const sentWants = new Set() // new peer, give them the full wantlist @@ -257,7 +511,11 @@ export class WantList implements Startable { } } - disconnected (peerId: PeerId): void { + /** + * Invoked when the network topology notices peer that supports Bitswap has + * disconnected + */ + peerDisconnected (peerId: PeerId): void { this.peers.delete(peerId) } diff --git a/packages/bitswap/test/bitswap.spec.ts b/packages/bitswap/test/bitswap.spec.ts index 3e03729c..4f3117fb 100644 --- a/packages/bitswap/test/bitswap.spec.ts +++ b/packages/bitswap/test/bitswap.spec.ts @@ -17,6 +17,7 @@ import { stubInterface } from 'sinon-ts' import { Bitswap } from '../src/bitswap.js' import { BitswapMessage, BlockPresenceType } from '../src/pb/message.js' import { cidToPrefix } from '../src/utils/cid-prefix.js' +import type { BitswapMessageEventDetail } from '../src/network.js' import type { Routing } from '@helia/interface/routing' import type { Connection, Libp2p, PeerId } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' @@ -83,37 +84,17 @@ describe('bitswap', () => { }) // providers found via routing - const providers = [{ - id: await createEd25519PeerId(), - multiaddrs: [ - multiaddr('/ip4/41.41.41.41/tcp/1234') - ], - protocols: ['transport-bitswap'] - }, { - id: await createEd25519PeerId(), - multiaddrs: [ - multiaddr('/ip4/42.42.42.42/tcp/1234') - ], - protocols: ['transport-bitswap'] - }, { - id: await createEd25519PeerId(), - multiaddrs: [ - multiaddr('/ip4/43.43.43.43/tcp/1234') - ], - protocols: ['transport-bitswap'] - }, { - id: await createEd25519PeerId(), - multiaddrs: [ - multiaddr('/ip4/44.44.44.44/tcp/1234') - ], - protocols: ['transport-bitswap'] - }, { - id: await createEd25519PeerId(), - multiaddrs: [ - multiaddr('/ip4/45.45.45.45/tcp/1234') - ], - protocols: ['transport-bitswap'] - }] + const providers = await Promise.all( + new Array(10).fill(0).map(async (_, i) => { + return { + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr(`/ip4/4${i}.4${i}.4${i}.4${i}/tcp/${1234 + i}`) + ], + protocols: ['transport-bitswap'] + } + }) + ) components.routing.findProviders.withArgs(cid).returns((async function * () { yield * providers @@ -161,6 +142,14 @@ describe('bitswap', () => { blocks: [], pendingBytes: 0 }) + stubPeerResponse(components.libp2p, providers[5].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.HaveBlock + }], + blocks: [], + pendingBytes: 0 + }) const session = await bitswap.createSession(cid) expect(session.peers.size).to.equal(DEFAULT_SESSION_MIN_PROVIDERS) @@ -289,67 +278,50 @@ describe('bitswap', () => { }) describe('want', () => { - it('should want a block that is in the blockstore', async () => { - await components.blockstore.put(cid, block) - - const b = await bitswap.want(cid) - - expect(b).to.equalBytes(block) - }) - it('should want a block that is available on the network', async () => { const remotePeer = await createEd25519PeerId() - const wantBlockSpy = Sinon.spy(bitswap.notifications, 'wantBlock') - const addToWantlistSpy = Sinon.spy(bitswap.wantList, 'wantBlocks') const findProvsSpy = Sinon.spy(bitswap.network, 'findAndConnect') const p = bitswap.want(cid) // provider sends message - await bitswap._receiveMessage(remotePeer, { - blocks: [{ - prefix: cidToPrefix(cid), - data: block - }], - blockPresences: [], - pendingBytes: 0 + bitswap.network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [{ + prefix: cidToPrefix(cid), + data: block + }], + blockPresences: [], + pendingBytes: 0 + } + } }) const b = await p // should have added cid to wantlist and searched for providers - expect(addToWantlistSpy.called).to.be.true() expect(findProvsSpy.called).to.be.true() // should have cancelled the notification request - expect(wantBlockSpy.called).to.be.true() - expect(wantBlockSpy.getCall(0)).to.have.nested.property('args[1].signal.aborted', true) - expect(b).to.equalBytes(block) }) it('should abort wanting a block that is not available on the network', async () => { - const wantBlockSpy = Sinon.spy(bitswap.notifications, 'wantBlock') - const p = bitswap.want(cid, { signal: AbortSignal.timeout(100) }) await expect(p).to.eventually.be.rejected - .with.property('code', 'ERR_ABORTED') - - // should have cancelled the notification request - expect(wantBlockSpy.called).to.be.true() - expect(wantBlockSpy.getCall(0)).to.have.nested.property('args[1].signal.aborted', true) + .with.property('code', 'ABORT_ERR') }) it('should notify peers we have a block', async () => { - const wantEventListener = bitswap.notifications.wantBlock(cid) const receivedBlockSpy = Sinon.spy(bitswap.peerWantLists, 'receivedBlock') await bitswap.notify(cid, block) - await expect(wantEventListener).to.eventually.deep.equal(block) expect(receivedBlockSpy.called).to.be.true() }) }) @@ -364,13 +336,18 @@ describe('bitswap', () => { expect(bitswap.getWantlist().map(w => w.cid)).to.include(cid) // provider sends message - await bitswap._receiveMessage(remotePeer, { - blocks: [{ - prefix: cidToPrefix(cid), - data: block - }], - blockPresences: [], - pendingBytes: 0 + bitswap.network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [{ + prefix: cidToPrefix(cid), + data: block + }], + blockPresences: [], + pendingBytes: 0 + } + } }) const b = await p @@ -379,7 +356,7 @@ describe('bitswap', () => { expect(b).to.equalBytes(block) }) - it('should remove CIDs from the wantlist when the want is cancelled', async () => { + it('should remove CIDs from the wantlist when the want is aborted', async () => { expect(bitswap.getWantlist()).to.be.empty() const p = bitswap.want(cid, { @@ -389,7 +366,7 @@ describe('bitswap', () => { expect(bitswap.getWantlist().map(w => w.cid)).to.include(cid) await expect(p).to.eventually.be.rejected - .with.property('code', 'ERR_ABORTED') + .with.property('code', 'ABORT_ERR') expect(bitswap.getWantlist()).to.be.empty() }) @@ -402,17 +379,23 @@ describe('bitswap', () => { // don't have this peer yet expect(bitswap.getPeerWantlist(remotePeer)).to.be.undefined() - await bitswap.peerWantLists.messageReceived(remotePeer, { - wantlist: { - full: false, - entries: [{ - cid: cid.bytes, - priority: 100 - }] - }, - blockPresences: [], - blocks: [], - pendingBytes: 0 + // peers sends message with wantlist + bitswap.network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + wantlist: { + full: false, + entries: [{ + cid: cid.bytes, + priority: 100 + }] + }, + blockPresences: [], + blocks: [], + pendingBytes: 0 + } + } }) expect(bitswap.getPeerWantlist(remotePeer)?.map(entry => entry.cid)).to.deep.equal([cid]) diff --git a/packages/bitswap/test/network.spec.ts b/packages/bitswap/test/network.spec.ts index 87f7fb7e..ad958e4a 100644 --- a/packages/bitswap/test/network.spec.ts +++ b/packages/bitswap/test/network.spec.ts @@ -1,3 +1,4 @@ +import { CustomEvent, isPeerId } from '@libp2p/interface' import { mockStream } from '@libp2p/interface-compliance-tests/mocks' import { defaultLogger } from '@libp2p/logger' import { createEd25519PeerId } from '@libp2p/peer-id-factory' @@ -18,7 +19,7 @@ import { Network } from '../src/network.js' import { BitswapMessage, BlockPresenceType } from '../src/pb/message.js' import { cidToPrefix } from '../src/utils/cid-prefix.js' import type { Routing } from '@helia/interface/routing' -import type { Connection, Libp2p, PeerId } from '@libp2p/interface' +import type { Connection, Libp2p, PeerId, IdentifyResult } from '@libp2p/interface' interface StubbedNetworkComponents { routing: StubbedInstance @@ -254,6 +255,36 @@ describe('network', () => { yield * providers })()) + components.libp2p.dial.callsFake(async (peerId) => { + // fake a network delay + await delay(100) + + const connection = stubInterface() + + // simulate identify having run + setTimeout(() => { + const call = components.libp2p.addEventListener.getCall(0) + + expect(call.args[0]).to.equal('peer:identify') + const callback = call.args[1] + + if (isPeerId(peerId) && typeof callback === 'function') { + callback(new CustomEvent('peer:identify', { + detail: { + peerId, + protocols: [ + BITSWAP_120 + ], + listenAddrs: [], + connection + } + })) + } + }, 100) + + return connection + }) + await network.findAndConnect(cid) expect(components.libp2p.dial.calledWith(peerId)).to.be.true() @@ -283,6 +314,36 @@ describe('network', () => { yield * providers })()) + components.libp2p.dial.callsFake(async (peerId) => { + // fake a network delay + await delay(100) + + const connection = stubInterface() + + // simulate identify having run + setTimeout(() => { + const call = components.libp2p.addEventListener.getCall(0) + + expect(call.args[0]).to.equal('peer:identify') + const callback = call.args[1] + + if (isPeerId(peerId) && typeof callback === 'function') { + callback(new CustomEvent('peer:identify', { + detail: { + peerId, + protocols: [ + BITSWAP_120 + ], + listenAddrs: [], + connection + } + })) + } + }, 100) + + return connection + }) + await network.findAndConnect(cid) expect(components.libp2p.dial.calledWith(peerId)).to.be.true() diff --git a/packages/bitswap/test/notifications.spec.ts b/packages/bitswap/test/notifications.spec.ts deleted file mode 100644 index df1d6166..00000000 --- a/packages/bitswap/test/notifications.spec.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { defaultLogger } from '@libp2p/logger' -import { createEd25519PeerId } from '@libp2p/peer-id-factory' -import { expect } from 'aegir/chai' -import { CID } from 'multiformats/cid' -import { pEvent } from 'p-event' -import { Notifications, doNotHaveEvent, haveEvent } from '../src/notifications.js' - -describe('notifications', () => { - let notifications: Notifications - - before(() => { - notifications = new Notifications({ - logger: defaultLogger() - }) - }) - - it('should notify wants after receiving a block', async () => { - const peerId = await createEd25519PeerId() - const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - const data = Uint8Array.from([0, 1, 2, 3, 4]) - - const p = notifications.wantBlock(cid) - - notifications.receivedBlock(cid, data, peerId) - - const block = await p - - expect(block).to.equalBytes(data) - }) - - it('should notify wants after unwanting a block', async () => { - const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - const p = notifications.wantBlock(cid) - - notifications.unwantBlock(cid) - - await expect(p).to.eventually.rejected.with.property('code', 'ERR_UNWANTED') - }) - - it('should notify wants aborting wanting a block', async () => { - const signal = AbortSignal.timeout(100) - const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - const p = notifications.wantBlock(cid, { - signal - }) - - await expect(p).to.eventually.rejected.with.property('code', 'ERR_ABORTED') - }) - - it('should notify on have', async () => { - const peerId = await createEd25519PeerId() - const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - - const p = pEvent(notifications, haveEvent(cid)) - - notifications.haveBlock(cid, peerId) - - await expect(p).to.eventually.equal(peerId) - }) - - it('should notify on do not have', async () => { - const peerId = await createEd25519PeerId() - const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - - const p = pEvent(notifications, doNotHaveEvent(cid)) - - notifications.doNotHaveBlock(cid, peerId) - - await expect(p).to.eventually.equal(peerId) - }) -}) diff --git a/packages/bitswap/test/peer-want-list.spec.ts b/packages/bitswap/test/peer-want-list.spec.ts index 36dddeb8..2ed3e0ed 100644 --- a/packages/bitswap/test/peer-want-list.spec.ts +++ b/packages/bitswap/test/peer-want-list.spec.ts @@ -5,31 +5,43 @@ import { MemoryBlockstore } from 'blockstore-core' import delay from 'delay' import { CID } from 'multiformats/cid' import pRetry from 'p-retry' -import { stubInterface, type StubbedInstance } from 'sinon-ts' -import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../src/constants.js' +import Sinon from 'sinon' +import { stubInterface } from 'sinon-ts' +import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK, DEFAULT_MESSAGE_SEND_DELAY } from '../src/constants.js' +import { Network } from '../src/network.js' import { BlockPresenceType, WantType } from '../src/pb/message.js' import { PeerWantLists } from '../src/peer-want-lists/index.js' import ve from '../src/utils/varint-encoder.js' -import type { Network } from '../src/network.js' -import type { ComponentLogger, PeerId } from '@libp2p/interface' +import type { Routing } from '@helia/interface' +import type { Libp2p, ComponentLogger, PeerId } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' interface PeerWantListsComponentStubs { peerId: PeerId blockstore: Blockstore - network: StubbedInstance + network: Network logger: ComponentLogger } describe('peer-want-lists', () => { let components: PeerWantListsComponentStubs let wantLists: PeerWantLists + let network: Network beforeEach(async () => { + const logger = defaultLogger() + network = new Network({ + routing: stubInterface(), + logger, + libp2p: stubInterface({ + getConnections: () => [] + }) + }) + components = { peerId: await createEd25519PeerId(), blockstore: new MemoryBlockstore(), - network: stubInterface(), + network, logger: defaultLogger() } @@ -43,16 +55,21 @@ describe('peer-want-lists', () => { const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - full: true, - entries: [{ - cid: cid.bytes, - priority: 1 - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + full: true, + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } } }) @@ -72,15 +89,20 @@ describe('peer-want-lists', () => { const cid2 = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') // first wantlist - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid1.bytes, - priority: 1 - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid1.bytes, + priority: 1 + }] + } + } } }) @@ -88,16 +110,21 @@ describe('peer-want-lists', () => { expect(entries?.map(entry => entry.cid.toString())).to.include(cid1.toString()) - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - full: true, - entries: [{ - cid: cid2.bytes, - priority: 1 - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + full: true, + entries: [{ + cid: cid2.bytes, + priority: 1 + }] + } + } } }) @@ -115,15 +142,20 @@ describe('peer-want-lists', () => { const cid2 = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') // first wantlist - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid1.bytes, - priority: 1 - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid1.bytes, + priority: 1 + }] + } + } } }) @@ -131,15 +163,20 @@ describe('peer-want-lists', () => { expect(entries?.map(entry => entry.cid.toString())).to.include(cid1.toString()) - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid2.bytes, - priority: 1 - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid2.bytes, + priority: 1 + }] + } + } } }) @@ -153,16 +190,21 @@ describe('peer-want-lists', () => { it('should record the amount of incoming data', async () => { const remotePeer = await createEd25519PeerId() - await wantLists.messageReceived(remotePeer, { - blocks: [{ - prefix: Uint8Array.from([0, 1, 2, 3, 4]), - data: Uint8Array.from([0, 1, 2, 3, 4]) - }, { - prefix: Uint8Array.from([0, 1, 2]), - data: Uint8Array.from([0, 1, 2]) - }], - blockPresences: [], - pendingBytes: 0 + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [{ + prefix: Uint8Array.from([0, 1, 2, 3, 4]), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + prefix: Uint8Array.from([0, 1, 2]), + data: Uint8Array.from([0, 1, 2]) + }], + blockPresences: [], + pendingBytes: 0 + } + } }) const ledger = wantLists.ledgerForPeer(remotePeer) @@ -171,6 +213,7 @@ describe('peer-want-lists', () => { }) it('should send requested blocks to peer', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() const remotePeer = await createEd25519PeerId() const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') @@ -179,26 +222,32 @@ describe('peer-want-lists', () => { // we have block await components.blockstore.put(cid, block) - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1 - }] + // incoming message + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } } }) // wait for network send await pRetry(() => { - if (!components.network.sendMessage.called) { + if (!sendMessageStub.called) { throw new Error('Network message not sent') } }) - const message = components.network.sendMessage.getCall(0).args[1] + const message = sendMessageStub.getCall(0).args[1] expect(message.blocks).to.have.lengthOf(1) expect(message.blocks?.[0].data).to.equalBytes(block) @@ -207,12 +256,13 @@ describe('peer-want-lists', () => { ])) // have to wait for network send - await delay(1) + await delay(DEFAULT_MESSAGE_SEND_DELAY * 3) expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.not.include(cid.toString()) }) it('should send requested block presences to peer', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() const remotePeer = await createEd25519PeerId() const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') @@ -221,27 +271,32 @@ describe('peer-want-lists', () => { // we have block await components.blockstore.put(cid, block) - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1, - wantType: WantType.WantHave - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave + }] + } + } } }) // wait for network send await pRetry(() => { - if (!components.network.sendMessage.called) { + if (!sendMessageStub.called) { throw new Error('Network message not sent') } }) - const message = components.network.sendMessage.getCall(0).args[1] + const message = sendMessageStub.getCall(0).args[1] expect(message.blocks).to.be.empty('should not have sent blocks') expect(message.blockPresences).to.have.lengthOf(1) @@ -250,33 +305,39 @@ describe('peer-want-lists', () => { }) it('should send requested lack of block presences to peer', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() const remotePeer = await createEd25519PeerId() // CID for a block we don't have const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1, - wantType: WantType.WantBlock, - sendDontHave: true - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantBlock, + sendDontHave: true + }] + } + } } }) // wait for network send await pRetry(() => { - if (!components.network.sendMessage.called) { + if (!sendMessageStub.called) { throw new Error('Network message not sent') } }) - const message = components.network.sendMessage.getCall(0).args[1] + const message = sendMessageStub.getCall(0).args[1] expect(message.blocks).to.be.empty('should not have sent blocks') expect(message.blockPresences).to.have.lengthOf(1) @@ -285,6 +346,7 @@ describe('peer-want-lists', () => { }) it('should send requested blocks to peer when presence was requested but block size is less than maxSizeReplaceHasWithBlock', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() const remotePeer = await createEd25519PeerId() const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') @@ -293,27 +355,32 @@ describe('peer-want-lists', () => { // we have block await components.blockstore.put(cid, block) - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1, - wantType: WantType.WantHave - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave + }] + } + } } }) // wait for network send await pRetry(() => { - if (!components.network.sendMessage.called) { + if (!sendMessageStub.called) { throw new Error('Network message not sent') } }) - const message = components.network.sendMessage.getCall(0).args[1] + const message = sendMessageStub.getCall(0).args[1] expect(message.blockPresences).to.be.empty() expect(message.blocks).to.have.lengthOf(1) @@ -329,32 +396,38 @@ describe('peer-want-lists', () => { }) it('should send requested block presences to peer for blocks we don\'t have', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() const remotePeer = await createEd25519PeerId() const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1, - wantType: WantType.WantHave, - sendDontHave: true - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave, + sendDontHave: true + }] + } + } } }) // wait for network send await pRetry(() => { - if (!components.network.sendMessage.called) { + if (!sendMessageStub.called) { throw new Error('Network message not sent') } }) - const message = components.network.sendMessage.getCall(0).args[1] + const message = sendMessageStub.getCall(0).args[1] expect(message.blocks).to.be.empty('should not have sent blocks') expect(message.blockPresences).to.have.lengthOf(1) @@ -367,30 +440,40 @@ describe('peer-want-lists', () => { const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1 - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } } }) expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.include(cid.toString()) - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1, - cancel: true - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + cancel: true + }] + } + } } }) @@ -406,15 +489,20 @@ describe('peer-want-lists', () => { // we have block await components.blockstore.put(cid, block) - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1 - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } } }) @@ -432,36 +520,47 @@ describe('peer-want-lists', () => { expect(wantLists.peers()).to.be.empty() - await wantLists.messageReceived(remotePeer, { - blocks: [{ - prefix: Uint8Array.from([0, 1, 2, 3, 4]), - data: Uint8Array.from([0, 1, 2, 3, 4]) - }, { - prefix: Uint8Array.from([0, 1, 2]), - data: Uint8Array.from([0, 1, 2]) - }], - blockPresences: [], - pendingBytes: 0 + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [{ + prefix: Uint8Array.from([0, 1, 2, 3, 4]), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + prefix: Uint8Array.from([0, 1, 2]), + data: Uint8Array.from([0, 1, 2]) + }], + blockPresences: [], + pendingBytes: 0 + } + } }) expect(wantLists.peers().map(p => p.toString())).to.include(remotePeer.toString()) }) it('should send requested blocks to peer when they are received', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() const remotePeer = await createEd25519PeerId() const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') const block = Uint8Array.from([0, 1, 2, 3, 4]) - await wantLists.messageReceived(remotePeer, { - blocks: [], - blockPresences: [], - pendingBytes: 0, - wantlist: { - entries: [{ - cid: cid.bytes, - priority: 1 - }] + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } } }) @@ -475,12 +574,12 @@ describe('peer-want-lists', () => { // wait for network send await pRetry(() => { - if (!components.network.sendMessage.called) { + if (!sendMessageStub.called) { throw new Error('Network message not sent') } }) - const message = components.network.sendMessage.getCall(0).args[1] + const message = sendMessageStub.getCall(0).args[1] expect(message.blocks).to.have.lengthOf(1) expect(message.blocks?.[0].data).to.equalBytes(block) @@ -496,6 +595,6 @@ describe('peer-want-lists', () => { // should only have sent one message await delay(100) - expect(components.network.sendMessage.callCount).to.equal(1) + expect(sendMessageStub.callCount).to.equal(1) }) }) diff --git a/packages/bitswap/test/session.spec.ts b/packages/bitswap/test/session.spec.ts index efb77043..f91c07b2 100644 --- a/packages/bitswap/test/session.spec.ts +++ b/packages/bitswap/test/session.spec.ts @@ -1,4 +1,5 @@ import { defaultLogger } from '@libp2p/logger' +import { PeerMap } from '@libp2p/peer-collections' import { createEd25519PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' import { CID } from 'multiformats/cid' @@ -6,11 +7,9 @@ import { stubInterface, type StubbedInstance } from 'sinon-ts' import { createBitswapSession } from '../src/session.js' import type { BitswapSession } from '../src/index.js' import type { Network } from '../src/network.js' -import type { Notifications } from '../src/notifications.js' import type { WantList } from '../src/want-list.js' interface StubbedBitswapSessionComponents { - notifications: StubbedInstance network: StubbedInstance wantList: StubbedInstance } @@ -24,37 +23,53 @@ describe('session', () => { cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') components = { - notifications: stubInterface(), network: stubInterface(), - wantList: stubInterface() + wantList: stubInterface({ + peers: new PeerMap() + }) } - - session = createBitswapSession({ - ...components, - logger: defaultLogger() - }, { - root: cid - }) }) it('should only query session peers', async () => { const peerId = await createEd25519PeerId() const data = new Uint8Array([0, 1, 2, 3, 4]) - session.peers.add(peerId) + components.network.findProviders.returns(async function * () { + yield { + id: peerId, + multiaddrs: [], + protocols: [''] + } + }()) + + components.wantList.wantPresence.resolves({ + sender: peerId, + cid, + has: true + }) - components.notifications.wantBlock.resolves(data) + components.wantList.wantBlock.resolves({ + sender: peerId, + cid, + block: data + }) + + session = await createBitswapSession({ + ...components, + logger: defaultLogger() + }, { + root: cid, + queryConcurrency: 5, + minProviders: 1, + maxProviders: 3, + connectedPeers: [] + }) const p = session.want(cid) - expect(components.wantList.wantBlocks.called).to.be.true() - expect(components.wantList.wantBlocks.getCall(0)).to.have.nested.property('args[1].session', session.peers) + expect(components.wantList.wantBlock.called).to.be.true() + expect(components.wantList.wantBlock.getCall(0).args[1]?.peerId?.toString()).to.equal(peerId.toString()) await expect(p).to.eventually.deep.equal(data) }) - - it('should throw when wanting from an empty session', async () => { - await expect(session.want(cid)).to.eventually.be.rejected - .with.property('code', 'ERR_NO_SESSION_PEERS') - }) }) diff --git a/packages/bitswap/test/want-list.spec.ts b/packages/bitswap/test/want-list.spec.ts index 18c4d605..333fe6df 100644 --- a/packages/bitswap/test/want-list.spec.ts +++ b/packages/bitswap/test/want-list.spec.ts @@ -1,5 +1,5 @@ +import { matchPeerId } from '@libp2p/interface-compliance-tests/matchers' import { defaultLogger } from '@libp2p/logger' -import { PeerSet } from '@libp2p/peer-collections' import { createEd25519PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' import { CID } from 'multiformats/cid' @@ -38,7 +38,7 @@ describe('wantlist', () => { it('should add peers to peer list on connect', async () => { const peerId = await createEd25519PeerId() - await wantList.connected(peerId) + await wantList.peerConnected(peerId) expect(wantList.peers.has(peerId)).to.be.true() }) @@ -46,11 +46,11 @@ describe('wantlist', () => { it('should remove peers to peer list on disconnect', async () => { const peerId = await createEd25519PeerId() - await wantList.connected(peerId) + await wantList.peerConnected(peerId) expect(wantList.peers.has(peerId)).to.be.true() - wantList.disconnected(peerId) + wantList.peerDisconnected(peerId) expect(wantList.peers.has(peerId)).to.be.false() }) @@ -59,10 +59,14 @@ describe('wantlist', () => { const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') const peerId = await createEd25519PeerId() - await wantList.connected(peerId) + await wantList.peerConnected(peerId) - components.network.sendMessage.withArgs(peerId).resolves() - await wantList.wantBlocks([cid]) + components.network.sendMessage.withArgs(matchPeerId(peerId)) + + await expect(wantList.wantBlock(cid, { + signal: AbortSignal.timeout(500) + })).to.eventually.be.rejected + .with.property('code', 'ABORT_ERR') const sentToPeer = components.network.sendMessage.getCall(0).args[0] expect(sentToPeer.toString()).equal(peerId.toString()) @@ -74,43 +78,19 @@ describe('wantlist', () => { expect(sentMessage).to.have.nested.property('wantlist.entries[0].cancel', false) }) - it('should cancel wants', async () => { - const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') - const peerId = await createEd25519PeerId() - components.network.sendMessage.withArgs(peerId).resolves() - - await wantList.connected(peerId) - - await wantList.wantBlocks([cid]) - - expect([...wantList.wants.values()].find(want => want.cid.equals(cid))).to.be.ok() - - // now cancel - await wantList.cancelWants([cid]) - - const sentToPeer = components.network.sendMessage.getCall(1).args[0] - expect(sentToPeer.toString()).equal(peerId.toString()) - - const sentMessage = components.network.sendMessage.getCall(1).args[1] - expect(sentMessage).to.have.nested.property('wantlist.full', false) - expect(sentMessage).to.have.deep.nested.property('wantlist.entries[0].cid').that.equalBytes(cid.bytes) - expect(sentMessage).to.have.nested.property('wantlist.entries[0].wantType', WantType.WantBlock) - expect(sentMessage).to.have.nested.property('wantlist.entries[0].cancel', true) - - expect([...wantList.wants.values()].find(want => want.cid.equals(cid))).to.be.undefined() - }) - it('should not send session block wants to non-session peers', async () => { const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') const sessionPeer = await createEd25519PeerId() const nonSessionPeer = await createEd25519PeerId() - await wantList.connected(sessionPeer) - await wantList.connected(nonSessionPeer) + await wantList.peerConnected(sessionPeer) + await wantList.peerConnected(nonSessionPeer) - await wantList.wantBlocks([cid], { - session: new PeerSet([sessionPeer]) - }) + await expect(wantList.wantBlock(cid, { + peerId: sessionPeer, + signal: AbortSignal.timeout(500) + })).to.eventually.be.rejected + .with.property('code', 'ABORT_ERR') expect(components.network.sendMessage.callCount).to.equal(1) From 7eef3f1eab0a2268edc7a10dcfddc2deba312837 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 15 Feb 2024 11:42:14 +0000 Subject: [PATCH 18/27] chore: pass init to wantlist --- packages/bitswap/src/bitswap.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts index e9f4dc87..2ec01b54 100644 --- a/packages/bitswap/src/bitswap.ts +++ b/packages/bitswap/src/bitswap.ts @@ -61,7 +61,7 @@ export class Bitswap implements BitswapInterface { this.wantList = new WantList({ ...components, network: this.network - }) + }, init) } // TODO: remove me From aabc0f40a9c7fd34408f1eb1685ac1043a43a160 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 22 Feb 2024 18:34:51 +0000 Subject: [PATCH 19/27] chore: apply suggestions from code review Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> --- packages/bitswap/src/index.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/bitswap/src/index.ts b/packages/bitswap/src/index.ts index b7641b93..3e3c85b3 100644 --- a/packages/bitswap/src/index.ts +++ b/packages/bitswap/src/index.ts @@ -51,15 +51,15 @@ export interface WantListEntry { export interface CreateBitswapSessionOptions extends CreateSessionOptions { /** - * If true, query connected peers before searching for providers in the - * routing + * If true, query connected peers before searching for providers via + * Helia routers * * @default true */ queryConnectedPeers?: boolean /** - * If true, search for providers in the routing to query for the root CID + * If true, search for providers via Helia routers to query for the root CID * * @default true */ @@ -179,15 +179,17 @@ export interface BitswapOptions { sendBlocksConcurrency?: number /** - * When sending want list updates to peers, how many messages to send at once + * When sending blocks to peers, timeout after this many milliseconds. + * This is useful for preventing slow/large peer-connections from consuming + * your bandwidth/streams. * * @default 10000 */ sendBlocksTimeout?: number /** - * When a block is added to the blockstore and we are about to sending that - * block to peers who have it in their wantlist, wait this long before + * When a block is added to the blockstore and we are about to send that + * block to peers who have it in their wantlist, wait this many milliseconds before * queueing the send job in case more blocks are added that they want * * @default 10 From d7d6334036a61e246522885eb27afa9f6cfd3d4d Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 1 Mar 2024 08:32:25 +0000 Subject: [PATCH 20/27] chore: apply suggestions from code review Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> --- packages/interface/src/blocks.ts | 6 ++---- packages/utils/src/utils/networked-storage.ts | 7 ++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index 5511b7b0..a69d5d6d 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -64,8 +64,6 @@ ProgressOptions, ProgressOptions): Promise } @@ -110,7 +108,7 @@ export interface CreateSessionOptions > { + return typeof broker.retrieve === 'function' +} export const getCidBlockVerifierFunction = (cid: CID, hasher: MultihashHasher): Required['validateFn'] => { if (hasher == null) { throw new CodeError(`No hasher configured for multihash code 0x${cid.multihash.code.toString(16)}, please configure one. You can look up which hash this is at https://github.com/multiformats/multicodec/blob/master/table.csv`, 'ERR_UNKNOWN_HASH_ALG') @@ -246,9 +249,7 @@ async function raceBlockRetrievers (cid: CID, blockBrokers: BlockBroker[], hashe const retrievers: Array>> = [] for (const broker of blockBrokers) { - if (broker.retrieve != null) { - // @ts-expect-error retrieve may be undefined even though we've just - // checked that it isn't + if (isRetrievingBlockBroker(broker)) { retrievers.push(broker) } } From 0766e125870d7d76dbe3f6beb4c2eb21002ced53 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 1 Mar 2024 09:43:48 +0000 Subject: [PATCH 21/27] chore: fix import --- packages/bitswap/src/bitswap.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts index 2ec01b54..db817615 100644 --- a/packages/bitswap/src/bitswap.ts +++ b/packages/bitswap/src/bitswap.ts @@ -1,5 +1,5 @@ /* eslint-disable no-loop-func */ -import { DEFAULT_SESSION_MAX_PROVIDERS, DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_QUERY_CONCURRENCY } from '@helia/interface' +import { DEFAULT_SESSION_MAX_PROVIDERS, DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_PROVIDER_QUERY_CONCURRENCY } from '@helia/interface' import { setMaxListeners } from '@libp2p/interface' import { anySignal } from 'any-signal' import { Network } from './network.js' @@ -85,7 +85,7 @@ export class Bitswap implements BitswapInterface { logger: this.logger }, { root, - queryConcurrency: options?.queryConcurrency ?? DEFAULT_SESSION_QUERY_CONCURRENCY, + queryConcurrency: options?.providerQueryConcurrency ?? DEFAULT_SESSION_PROVIDER_QUERY_CONCURRENCY, minProviders, maxProviders, connectedPeers: options?.queryConnectedPeers !== false ? [...this.wantList.peers.keys()] : [], From cba358f5c41fc0b1e0f11322339d3679585baacb Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 1 Mar 2024 09:51:52 +0000 Subject: [PATCH 22/27] chore: deps --- packages/block-brokers/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/block-brokers/package.json b/packages/block-brokers/package.json index 80bbcd4b..92d63a52 100644 --- a/packages/block-brokers/package.json +++ b/packages/block-brokers/package.json @@ -57,7 +57,6 @@ "@helia/interface": "^4.0.1", "@libp2p/interface": "^1.1.4", "interface-blockstore": "^5.2.10", - "ipfs-bitswap": "^20.0.2", "multiformats": "^13.1.0", "progress-events": "^1.0.0" }, From bff535730878dfb9c9be2e0c3cc689fbab247abc Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 1 Mar 2024 10:10:12 +0000 Subject: [PATCH 23/27] chore: remove delay --- packages/bitswap/test/bitswap.spec.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/bitswap/test/bitswap.spec.ts b/packages/bitswap/test/bitswap.spec.ts index 4f3117fb..f8a4a630 100644 --- a/packages/bitswap/test/bitswap.spec.ts +++ b/packages/bitswap/test/bitswap.spec.ts @@ -12,6 +12,7 @@ import { duplexPair } from 'it-pair/duplex' import { pbStream } from 'it-protobuf-stream' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' +import pWaitFor from 'p-wait-for' import Sinon from 'sinon' import { stubInterface } from 'sinon-ts' import { Bitswap } from '../src/bitswap.js' @@ -162,7 +163,9 @@ describe('bitswap', () => { expect(providers[0].id.equals(components.libp2p.dialProtocol.getCall(1).args[0].toString())).to.be.true() // the query continues after the session is ready - await delay(100) + await pWaitFor(() => { + return session.peers.size === DEFAULT_SESSION_MAX_PROVIDERS + }) // should have continued querying until we reach DEFAULT_SESSION_MAX_PROVIDERS expect(providers[1].id.equals(components.libp2p.dialProtocol.getCall(2).args[0].toString())).to.be.true() From 532bf9fa4c6c7d02d1acbdbad2ae9195eb35fe1a Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 1 Mar 2024 10:10:45 +0000 Subject: [PATCH 24/27] chore: deps --- packages/bitswap/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/bitswap/package.json b/packages/bitswap/package.json index c47a8f7b..aa352129 100644 --- a/packages/bitswap/package.json +++ b/packages/bitswap/package.json @@ -188,6 +188,7 @@ "it-protobuf-stream": "^1.1.2", "p-event": "^6.0.0", "p-retry": "^6.2.0", + "p-wait-for": "^5.0.2", "protons": "^7.0.2", "sinon": "^17.0.1", "sinon-ts": "^2.0.0" From 99cf45490170f816f3ed6ef1c086280bf554103f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 4 Apr 2024 12:25:32 +0100 Subject: [PATCH 25/27] chore: remove unused method --- packages/bitswap/src/bitswap.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts index db817615..d9af8b6e 100644 --- a/packages/bitswap/src/bitswap.ts +++ b/packages/bitswap/src/bitswap.ts @@ -64,17 +64,6 @@ export class Bitswap implements BitswapInterface { }, init) } - // TODO: remove me - _updateReceiveCounters (peerId: PeerId, data: Uint8Array, exists: boolean): void { - this.stats.updateBlocksReceived(1, peerId) - this.stats.updateDataReceived(data.byteLength, peerId) - - if (exists) { - this.stats.updateDuplicateBlocksReceived(1, peerId) - this.stats.updateDuplicateDataReceived(data.byteLength, peerId) - } - } - async createSession (root: CID, options?: CreateBitswapSessionOptions): Promise { const minProviders = options?.minProviders ?? DEFAULT_SESSION_MIN_PROVIDERS const maxProviders = options?.maxProviders ?? DEFAULT_SESSION_MAX_PROVIDERS From 13305043491248669ceaf87c7a48b968ad27fd11 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 4 Apr 2024 12:26:14 +0100 Subject: [PATCH 26/27] chore: remove unused field --- packages/bitswap/src/bitswap.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts index d9af8b6e..0c70ed71 100644 --- a/packages/bitswap/src/bitswap.ts +++ b/packages/bitswap/src/bitswap.ts @@ -37,12 +37,10 @@ export class Bitswap implements BitswapInterface { public blockstore: Blockstore public peerWantLists: PeerWantLists public wantList: WantList - public status: 'starting' | 'started' | 'stopping' | 'stopped' constructor (components: BitswapComponents, init: BitswapOptions = {}) { this.logger = components.logger this.log = components.logger.forComponent('helia:bitswap') - this.status = 'stopped' this.blockstore = components.blockstore // report stats to libp2p metrics @@ -140,21 +138,15 @@ export class Bitswap implements BitswapInterface { * Start the bitswap node */ async start (): Promise { - this.status = 'starting' - this.wantList.start() await this.network.start() - this.status = 'started' } /** * Stop the bitswap node */ async stop (): Promise { - this.status = 'stopping' - this.wantList.stop() await this.network.stop() - this.status = 'stopped' } } From d576fad6a9b5fa0e9017450b9fe09dad93aecb2d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 4 Apr 2024 12:38:00 +0100 Subject: [PATCH 27/27] chore: update docs --- packages/bitswap/README.md | 17 +++++++++++++++++ packages/bitswap/src/index.ts | 12 +++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/bitswap/README.md b/packages/bitswap/README.md index fc110943..163a5cec 100644 --- a/packages/bitswap/README.md +++ b/packages/bitswap/README.md @@ -7,8 +7,25 @@ # About + + This module implements the [Bitswap protocol](https://docs.ipfs.tech/concepts/bitswap/) in TypeScript. +It supersedes the older [ipfs-bitswap](https://www.npmjs.com/package/ipfs-bitswap) module with the aim of being smaller, faster, better integrated with libp2p/helia, having fewer dependencies and using standard JavaScript instead of Node.js APIs. + # Install ```console diff --git a/packages/bitswap/src/index.ts b/packages/bitswap/src/index.ts index 3e3c85b3..a2ff51c9 100644 --- a/packages/bitswap/src/index.ts +++ b/packages/bitswap/src/index.ts @@ -2,6 +2,8 @@ * @packageDocumentation * * This module implements the [Bitswap protocol](https://docs.ipfs.tech/concepts/bitswap/) in TypeScript. + * + * It supersedes the older [ipfs-bitswap](https://www.npmjs.com/package/ipfs-bitswap) module with the aim of being smaller, faster, better integrated with libp2p/helia, having fewer dependencies and using standard JavaScript instead of Node.js APIs. */ import { Bitswap as BitswapClass } from './bitswap.js' @@ -188,8 +190,8 @@ export interface BitswapOptions { sendBlocksTimeout?: number /** - * When a block is added to the blockstore and we are about to send that - * block to peers who have it in their wantlist, wait this many milliseconds before + * When a block is added to the blockstore and we are about to send that block + * to peers who have it in their wantlist, wait this many milliseconds before * queueing the send job in case more blocks are added that they want * * @default 10 @@ -197,9 +199,9 @@ export interface BitswapOptions { sendBlocksDebounce?: number /** - * If the client sends a want-have, and the engine has the corresponding - * block, we check the size of the block and if it's small enough we send the - * block itself, rather than sending a HAVE. + * If the client sends a want-have, and we have the corresponding block, we + * check the size of the block and if it's small enough we send the block + * itself, rather than sending a HAVE. * * This defines the maximum size up to which we replace a HAVE with a block. *