diff --git a/.release-please-manifest.json b/.release-please-manifest.json index fabe804f..f37d28ed 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,5 +1,6 @@ { "packages/block-brokers": "2.0.3", + "packages/bitswap": "0.0.0", "packages/car": "3.1.2", "packages/dag-cbor": "3.0.2", "packages/dag-json": "3.0.2", diff --git a/.release-please.json b/.release-please.json index 10d16aa8..6598baff 100644 --- a/.release-please.json +++ b/.release-please.json @@ -10,6 +10,7 @@ ], "packages": { "packages/block-brokers": {}, + "packages/bitswap": {}, "packages/car": {}, "packages/dag-cbor": {}, "packages/dag-json": {}, diff --git a/packages/bitswap/.aegir.js b/packages/bitswap/.aegir.js new file mode 100644 index 00000000..b7d54c2f --- /dev/null +++ b/packages/bitswap/.aegir.js @@ -0,0 +1,7 @@ + +/** @type {import('aegir').PartialOptions} */ +export default { + build: { + bundlesizeMax: '33KB' + } +} diff --git a/packages/bitswap/CHANGELOG.md b/packages/bitswap/CHANGELOG.md new file mode 100644 index 00000000..f9d785c9 --- /dev/null +++ b/packages/bitswap/CHANGELOG.md @@ -0,0 +1,902 @@ +## [20.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v19.0.2...v20.0.0) (2023-11-30) + + +### ⚠ BREAKING CHANGES + +* requires libp2p v1 + +### Dependencies + +* update libp2p to v1 ([#610](https://github.com/ipfs/js-ipfs-bitswap/issues/610)) ([9f8258d](https://github.com/ipfs/js-ipfs-bitswap/commit/9f8258da440f6b2c5064687bf10136c785344234)) + +## [19.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v19.0.1...v19.0.2) (2023-11-04) + + +### Dependencies + +* **dev:** bump sinon from 16.1.3 to 17.0.1 ([#606](https://github.com/ipfs/js-ipfs-bitswap/issues/606)) ([01f8738](https://github.com/ipfs/js-ipfs-bitswap/commit/01f8738df272bed69bcb2714dbe80c3e28f4d7b6)) + +## [19.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v19.0.0...v19.0.1) (2023-10-09) + + +### Trivial Changes + +* add or force update .github/workflows/js-test-and-release.yml ([#598](https://github.com/ipfs/js-ipfs-bitswap/issues/598)) ([061acfa](https://github.com/ipfs/js-ipfs-bitswap/commit/061acfa2ffb8e1893bb3a26ebd320be20973e322)) +* delete templates [skip ci] ([#597](https://github.com/ipfs/js-ipfs-bitswap/issues/597)) ([0d56b0a](https://github.com/ipfs/js-ipfs-bitswap/commit/0d56b0aaa6de9a58f2b73e72784ad49ba4b2db45)) + + +### Dependencies + +* **dev:** bump aegir from 40.0.13 to 41.0.0 ([#601](https://github.com/ipfs/js-ipfs-bitswap/issues/601)) ([a510fdc](https://github.com/ipfs/js-ipfs-bitswap/commit/a510fdc7b149b94bc3043c4f5b25700270a5408f)) +* **dev:** bump sinon from 15.2.0 to 16.1.0 ([#602](https://github.com/ipfs/js-ipfs-bitswap/issues/602)) ([ac858e7](https://github.com/ipfs/js-ipfs-bitswap/commit/ac858e7d8edcf965da150d89adc0f7a470f2899a)) + +## [19.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v18.0.3...v19.0.0) (2023-08-05) + + +### ⚠ BREAKING CHANGES + +* requires libp2p@0.46.x or later + +### Dependencies + +* update to libp2p 0.46.x ([#596](https://github.com/ipfs/js-ipfs-bitswap/issues/596)) ([be8fe06](https://github.com/ipfs/js-ipfs-bitswap/commit/be8fe06bb10d6d940ac51c56b30c80154469673b)) + +## [18.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v18.0.2...v18.0.3) (2023-07-27) + + +### Dependencies + +* **dev:** bump aegir from 39.0.13 to 40.0.1 ([#585](https://github.com/ipfs/js-ipfs-bitswap/issues/585)) ([09755bd](https://github.com/ipfs/js-ipfs-bitswap/commit/09755bd46a4fa7599ffdabc35d3ac7d8115abc07)) + +## [18.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v18.0.1...v18.0.2) (2023-07-27) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([79974a5](https://github.com/ipfs/js-ipfs-bitswap/commit/79974a5b165d84b878f25bb11ac63639d4e3e093)) +* Update .github/workflows/stale.yml [skip ci] ([63f3993](https://github.com/ipfs/js-ipfs-bitswap/commit/63f39935cfd560319b6eaf117e539bd018316dc7)) + + +### Dependencies + +* **dev:** bump delay from 5.0.0 to 6.0.0 ([#580](https://github.com/ipfs/js-ipfs-bitswap/issues/580)) ([d796ebc](https://github.com/ipfs/js-ipfs-bitswap/commit/d796ebcd7eb0267cc2709017ec06faf16a4ca2bb)) +* **dev:** bump p-event from 5.0.1 to 6.0.0 ([#582](https://github.com/ipfs/js-ipfs-bitswap/issues/582)) ([ae8fd6f](https://github.com/ipfs/js-ipfs-bitswap/commit/ae8fd6f16bdc49779b78fba3a22975153c138f7e)) + +## [18.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v18.0.0...v18.0.1) (2023-05-22) + + +### Bug Fixes + +* add events dep ([#581](https://github.com/ipfs/js-ipfs-bitswap/issues/581)) ([d26cd16](https://github.com/ipfs/js-ipfs-bitswap/commit/d26cd1642e6abc6c0055451a99ac2daf5b1ad341)) + +## [18.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v17.0.3...v18.0.0) (2023-05-19) + + +### ⚠ BREAKING CHANGES + +* bump libp2p from 0.43.4 to 0.45.1 (#579) + +### Dependencies + +* bump libp2p from 0.43.4 to 0.45.1 ([#579](https://github.com/ipfs/js-ipfs-bitswap/issues/579)) ([90691b9](https://github.com/ipfs/js-ipfs-bitswap/commit/90691b911ea3fedbc030fc2582939d8b6a7e80fc)) + +## [17.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v17.0.2...v17.0.3) (2023-05-19) + + +### Dependencies + +* **dev:** bump aegir from 38.1.8 to 39.0.7 ([#576](https://github.com/ipfs/js-ipfs-bitswap/issues/576)) ([1868cac](https://github.com/ipfs/js-ipfs-bitswap/commit/1868cac8c0934286b91ad05e4b1a38481a3b49b8)) + +## [17.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v17.0.1...v17.0.2) (2023-04-13) + + +### Bug Fixes + +* increase default stream limit ([#550](https://github.com/ipfs/js-ipfs-bitswap/issues/550)) ([3484be0](https://github.com/ipfs/js-ipfs-bitswap/commit/3484be038321e9c2828bded225f4c6dc94a4d5d9)) + +## [17.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v17.0.0...v17.0.1) (2023-04-04) + + +### Bug Fixes + +* add missing events dep ([#548](https://github.com/ipfs/js-ipfs-bitswap/issues/548)) ([eaf862a](https://github.com/ipfs/js-ipfs-bitswap/commit/eaf862a54b6f498431e1bac33c8e0b94e9b43e01)) + +## [17.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v16.0.0...v17.0.0) (2023-03-13) + + +### ⚠ BREAKING CHANGES + +* `.get`, `.getMany`, `.put` and `.putMany` are no longer part of the `Bitswap` interface - instead call `.want` and `.notify` + +### Features + +* simplify bitswap interface, add progress handlers ([#527](https://github.com/ipfs/js-ipfs-bitswap/issues/527)) ([1f31995](https://github.com/ipfs/js-ipfs-bitswap/commit/1f3199505ac53e3c16cb8ea713d2279fbe69acb1)) + +## [16.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v15.0.2...v16.0.0) (2023-02-13) + + +### ⚠ BREAKING CHANGES + +* this module is now typescript + +### Features + +* convert to typescript ([#525](https://github.com/ipfs/js-ipfs-bitswap/issues/525)) ([11d9261](https://github.com/ipfs/js-ipfs-bitswap/commit/11d9261bf26c722a5fa28fc7c9d52c7090cb12a6)) + + +### Trivial Changes + +* update paths ([3874ec4](https://github.com/ipfs/js-ipfs-bitswap/commit/3874ec451fb714ae7e48a4a0b9f69a8187a82255)) + +## [15.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v15.0.1...v15.0.2) (2023-01-27) + + +### Bug Fixes + +* implement .has method from the blockstore interface ([#520](https://github.com/ipfs/js-ipfs-bitswap/issues/520)) ([6cd37ac](https://github.com/ipfs/js-ipfs-bitswap/commit/6cd37ac05a3e5a3e0c1f3e11c9ba73afea7c29d9)) + + +### Trivial Changes + +* remove rimraf as it is not used ([#521](https://github.com/ipfs/js-ipfs-bitswap/issues/521)) ([eac64fd](https://github.com/ipfs/js-ipfs-bitswap/commit/eac64fd1a202ddcbacf73e0e1d3f52c1286b0503)) + +## [15.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v15.0.0...v15.0.1) (2023-01-27) + + +### Dependencies + +* **dev:** bump @chainsafe/libp2p-noise from 10.2.0 to 11.0.0 ([#511](https://github.com/ipfs/js-ipfs-bitswap/issues/511)) ([584db6c](https://github.com/ipfs/js-ipfs-bitswap/commit/584db6c1e226e6f9a5415236612b03ec38523d14)) +* **dev:** bump aegir from 37.12.1 to 38.1.0 ([#513](https://github.com/ipfs/js-ipfs-bitswap/issues/513)) ([72d6a4c](https://github.com/ipfs/js-ipfs-bitswap/commit/72d6a4c48f49583a41967200e567656cef70a9fe)) + +## [15.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v14.0.0...v15.0.0) (2023-01-07) + + +### ⚠ BREAKING CHANGES + +* update multiformats to v11 (#509) + +### Dependencies + +* update multiformats to v11 ([#509](https://github.com/ipfs/js-ipfs-bitswap/issues/509)) ([09d4ff9](https://github.com/ipfs/js-ipfs-bitswap/commit/09d4ff948a9292df03c6205d2e9b3545e166509c)) + +## [14.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v13.0.0...v14.0.0) (2022-11-19) + + +### ⚠ BREAKING CHANGES + +* updates to the new metrics interface + +### Bug Fixes + +* Update to new metrics interface ([#502](https://github.com/ipfs/js-ipfs-bitswap/issues/502)) ([60a5e6c](https://github.com/ipfs/js-ipfs-bitswap/commit/60a5e6cdb3fdb06ef9c8b935b339d2d3e5f8ef07)) + +## [13.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.6...v13.0.0) (2022-10-18) + + +### ⚠ BREAKING CHANGES + +* updates to incompatible multiformats version + +### Dependencies + +* update multiformats to 10.x.x ([#490](https://github.com/ipfs/js-ipfs-bitswap/issues/490)) ([123f06b](https://github.com/ipfs/js-ipfs-bitswap/commit/123f06b372800e5a8a993cd959dcf216f296f320)) + +## [12.0.6](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.5...v12.0.6) (2022-09-21) + + +### Dependencies + +* update @multiformats/multiaddr to 11.0.0 ([#478](https://github.com/ipfs/js-ipfs-bitswap/issues/478)) ([259b69c](https://github.com/ipfs/js-ipfs-bitswap/commit/259b69cb863d63aa61a254a945805ec9a9bef78c)) + +## [12.0.5](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.4...v12.0.5) (2022-09-01) + + +### Bug Fixes + +* reset timeout controller when messages are received ([#474](https://github.com/ipfs/js-ipfs-bitswap/issues/474)) ([f6c6317](https://github.com/ipfs/js-ipfs-bitswap/commit/f6c6317c878a75a760e8d46d4b53e2530631d42b)) + + +### Trivial Changes + +* update project ([#473](https://github.com/ipfs/js-ipfs-bitswap/issues/473)) ([40376cf](https://github.com/ipfs/js-ipfs-bitswap/commit/40376cffd144583dd7f72644a32c79cd4ce5acd0)) + +## [12.0.4](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.3...v12.0.4) (2022-08-17) + + +### Bug Fixes + +* ensure stream is closed when protocol is incorrect ([#471](https://github.com/ipfs/js-ipfs-bitswap/issues/471)) ([2509772](https://github.com/ipfs/js-ipfs-bitswap/commit/2509772baa31bea6ad610a721a4eb31dfa6f125f)) + +## [12.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.2...v12.0.3) (2022-08-15) + + +### Bug Fixes + +* close streams after use ([#470](https://github.com/ipfs/js-ipfs-bitswap/issues/470)) ([a10e9e9](https://github.com/ipfs/js-ipfs-bitswap/commit/a10e9e921d50a4f50ab4e665e1be35f3e4767b56)) + + +### Dependencies + +* bump blockstore-core from 1.0.5 to 2.0.1 ([#469](https://github.com/ipfs/js-ipfs-bitswap/issues/469)) ([2c10911](https://github.com/ipfs/js-ipfs-bitswap/commit/2c109113da63ffb339501740a31b768d8053ea1e)) +* bump interface-blockstore from 2.0.3 to 3.0.0 ([#467](https://github.com/ipfs/js-ipfs-bitswap/issues/467)) ([2f238a9](https://github.com/ipfs/js-ipfs-bitswap/commit/2f238a9ca20eb5402a8f1e3b800668cf0d46d613)) +* **dev:** bump interface-datastore from 6.1.1 to 7.0.0 ([#468](https://github.com/ipfs/js-ipfs-bitswap/issues/468)) ([e7852d1](https://github.com/ipfs/js-ipfs-bitswap/commit/e7852d117b57acb81161a9e76925810407028d59)) + +## [12.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.1...v12.0.2) (2022-08-11) + + +### Bug Fixes + +* time out slow senders ([#455](https://github.com/ipfs/js-ipfs-bitswap/issues/455)) ([1a14c92](https://github.com/ipfs/js-ipfs-bitswap/commit/1a14c92347a1e18754b51b69bd5bbfe5b595e88d)) + + +### Trivial Changes + +* Update .github/workflows/stale.yml [skip ci] ([b590c92](https://github.com/ipfs/js-ipfs-bitswap/commit/b590c92f7102326cef251f07c1e6b9f5758d60d8)) +* update project config ([#466](https://github.com/ipfs/js-ipfs-bitswap/issues/466)) ([799e6b0](https://github.com/ipfs/js-ipfs-bitswap/commit/799e6b03df4785e2bf1664771a18d8fcf42fa654)) + + +### Dependencies + +* update protobufs, it-pipe, etc ([#465](https://github.com/ipfs/js-ipfs-bitswap/issues/465)) ([019d5e7](https://github.com/ipfs/js-ipfs-bitswap/commit/019d5e72ecbb2c91f48c3a3d808185d8b7e754d3)) + +## [12.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v12.0.0...v12.0.1) (2022-06-29) + + +### Trivial Changes + +* update deps ([#454](https://github.com/ipfs/js-ipfs-bitswap/issues/454)) ([7516593](https://github.com/ipfs/js-ipfs-bitswap/commit/751659351640756d82b9c7ee21bea57efacea877)) + +## [12.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.4...v12.0.0) (2022-06-28) + + +### ⚠ BREAKING CHANGES + +* uses libp2p with protocol stream limiting + +### Features + +* update libp2p deps ([#453](https://github.com/ipfs/js-ipfs-bitswap/issues/453)) ([2383088](https://github.com/ipfs/js-ipfs-bitswap/commit/2383088d6732ddae1cf3fe3470a832ab33013aad)) + +## [11.0.4](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.3...v11.0.4) (2022-06-24) + + +### Trivial Changes + +* remove unused dev dep ([#451](https://github.com/ipfs/js-ipfs-bitswap/issues/451)) ([34b19e0](https://github.com/ipfs/js-ipfs-bitswap/commit/34b19e0128d781a89742ca637388367638aefc63)) + +## [11.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.2...v11.0.3) (2022-06-23) + + +### Bug Fixes + +* iterate over connections instead of every peer in the peerstore ([#450](https://github.com/ipfs/js-ipfs-bitswap/issues/450)) ([dc9b126](https://github.com/ipfs/js-ipfs-bitswap/commit/dc9b12616a7c909063a23a8dcd71a08a379967bd)) + +### [11.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.1...v11.0.2) (2022-05-25) + + +### Trivial Changes + +* update libp2p interfaces ([#435](https://github.com/ipfs/js-ipfs-bitswap/issues/435)) ([c37ba88](https://github.com/ipfs/js-ipfs-bitswap/commit/c37ba88626bfb540844029f526c6f001f6667526)) + +### [11.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v11.0.0...v11.0.1) (2022-04-11) + + +### Bug Fixes + +* include dist folder in published package ([#430](https://github.com/ipfs/js-ipfs-bitswap/issues/430)) ([52d5fcc](https://github.com/ipfs/js-ipfs-bitswap/commit/52d5fcc3caeec6a1592e1bb422ae5afcbcab673d)) + +## [11.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v10.0.2...v11.0.0) (2022-04-07) + + +### ⚠ BREAKING CHANGES + +* this module is now ESM only + +### Features + +* update to typescript version of libp2p ([#428](https://github.com/ipfs/js-ipfs-bitswap/issues/428)) ([23d24ce](https://github.com/ipfs/js-ipfs-bitswap/commit/23d24ce295b90cd3fcb5b229b23258e6ff45d5c9)) + +### [10.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v10.0.1...v10.0.2) (2022-01-20) + + +### Bug Fixes + +* remove abort controller deps ([#402](https://github.com/ipfs/js-ipfs-bitswap/issues/402)) ([ef6c8ce](https://github.com/ipfs/js-ipfs-bitswap/commit/ef6c8ce03f53b27a4b64a4e7a3ab78f11a4dbccb)) + +### [10.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v10.0.0...v10.0.1) (2022-01-20) + + +### Bug Fixes + +* await network start and stop ([#415](https://github.com/ipfs/js-ipfs-bitswap/issues/415)) ([44dfcb5](https://github.com/ipfs/js-ipfs-bitswap/commit/44dfcb5dc40ed6ba8d3e4a5587d38634a37730c0)) + + +### Trivial Changes + +* switch to unified ci ([#408](https://github.com/ipfs/js-ipfs-bitswap/issues/408)) ([2d8e20f](https://github.com/ipfs/js-ipfs-bitswap/commit/2d8e20f6053c800f801060d042e3367dbbb97102)) + +# [10.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v9.0.0...v10.0.0) (2022-01-18) + + +### Features + +* libp2p async peerstore ([#413](https://github.com/ipfs/js-ipfs-bitswap/issues/413)) ([9f5fda9](https://github.com/ipfs/js-ipfs-bitswap/commit/9f5fda97903c4ffcd39d7affa887648df1169be3)) + + +### BREAKING CHANGES + +* peerstore methods are now all async + + + +# [9.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v8.0.0...v9.0.0) (2021-12-02) + + +### chore + +* update libp2p-interfaces ([#388](https://github.com/ipfs/js-ipfs-bitswap/issues/388)) ([c1bcae7](https://github.com/ipfs/js-ipfs-bitswap/commit/c1bcae752b48d3714f4ad696b9e6573b5ab4efa7)) + + +### BREAKING CHANGES + +* requires node 15+ + + + +# [8.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v7.0.1...v8.0.0) (2021-11-22) + + +### Features + +* update dht ([#384](https://github.com/ipfs/js-ipfs-bitswap/issues/384)) ([8d96a18](https://github.com/ipfs/js-ipfs-bitswap/commit/8d96a180a632a38b6470f696d2c0648dc5146dc5)) + + +### BREAKING CHANGES + +* uses 0.26.x of libp2p-kad-dht + + + +## [7.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v7.0.0...v7.0.1) (2021-11-19) + + +### Bug Fixes + +* encode cidv1 prefixes ([#383](https://github.com/ipfs/js-ipfs-bitswap/issues/383)) ([35be758](https://github.com/ipfs/js-ipfs-bitswap/commit/35be758dd1f3e5851ce23754bd12933783c16416)) + + + +# [7.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v6.0.2...v7.0.0) (2021-09-14) + + + +## [6.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v6.0.1...v6.0.2) (2021-09-10) + + +### chore + +* switch to esm ([#370](https://github.com/ipfs/js-ipfs-bitswap/issues/370)) ([63edf01](https://github.com/ipfs/js-ipfs-bitswap/commit/63edf01d1c3f15c049d48bfe7fda4b6efe3a3206)) + + +### BREAKING CHANGES + +* uses named exports only + + + +## [6.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v6.0.0...v6.0.1) (2021-08-23) + + + +# [6.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.6...v6.0.0) (2021-07-10) + + +### chore + +* update to new multiformats ([#340](https://github.com/ipfs/js-ipfs-bitswap/issues/340)) ([73bdd19](https://github.com/ipfs/js-ipfs-bitswap/commit/73bdd19dbe7e9c6ef557918f843ccdef92c859de)) + + +### BREAKING CHANGES + +* uses the CID class from the new multiformats module + + + +## [5.0.6](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.5...v5.0.6) (2021-06-22) + + + +## [5.0.5](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.4...v5.0.5) (2021-05-13) + + +### Bug Fixes + +* fixes unhandled promise rejection ([#337](https://github.com/ipfs/js-ipfs-bitswap/issues/337)) ([f41fd0b](https://github.com/ipfs/js-ipfs-bitswap/commit/f41fd0b4a60a945f71ac0ba3c2c1df659f4b3339)), closes [#332](https://github.com/ipfs/js-ipfs-bitswap/issues/332) + + + +## [5.0.4](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.3...v5.0.4) (2021-04-30) + + + +## [5.0.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.2...v5.0.3) (2021-04-20) + + +### Bug Fixes + +* specify pbjs root ([#323](https://github.com/ipfs/js-ipfs-bitswap/issues/323)) ([2bf0c2e](https://github.com/ipfs/js-ipfs-bitswap/commit/2bf0c2e51cb5ee63e88868e84ae67b4e3ee0ce9b)) + + + +## [5.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.1...v5.0.2) (2021-04-16) + + +### Bug Fixes + +* fix wrong type signature ([#304](https://github.com/ipfs/js-ipfs-bitswap/issues/304)) ([47fdb2a](https://github.com/ipfs/js-ipfs-bitswap/commit/47fdb2a8f8fc6142e9879869402401a65b04cb0a)) + + + +## [5.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v5.0.0...v5.0.1) (2021-03-10) + + +### Bug Fixes + +* fixes bignumber import for type gen ([#301](https://github.com/ipfs/js-ipfs-bitswap/issues/301)) ([5c09a2e](https://github.com/ipfs/js-ipfs-bitswap/commit/5c09a2ee20f438e33da71a061e662bfae3701c9d)) + + + +# [5.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v4.0.2...v5.0.0) (2021-03-09) + + +### Features + +* typedef generation & type checking ([#261](https://github.com/ipfs/js-ipfs-bitswap/issues/261)) ([fca78c8](https://github.com/ipfs/js-ipfs-bitswap/commit/fca78c8c501a92a9726eea0d5e6942cdd6cba983)) + + + +## [4.0.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v4.0.1...v4.0.2) (2021-01-29) + + + +## [4.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v4.0.0...v4.0.1) (2021-01-21) + + +### Bug Fixes + +* update provider multiaddrs before dial ([#286](https://github.com/ipfs/js-ipfs-bitswap/issues/286)) ([49cc66c](https://github.com/ipfs/js-ipfs-bitswap/commit/49cc66cf387a27c146f8f0a111c3dff90101f47a)) + + + +# [4.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v3.0.0...v4.0.0) (2020-11-06) + + + + +# [3.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v2.0.1...v3.0.0) (2020-08-24) + + +### Bug Fixes + +* replace node buffers with uint8arrays ([#251](https://github.com/ipfs/js-ipfs-bitswap/issues/251)) ([4f9d7cd](https://github.com/ipfs/js-ipfs-bitswap/commit/4f9d7cd)) + + +### BREAKING CHANGES + +* - All use of node Buffers have been replaced with Uint8Arrays +- All deps now use Uint8Arrays in place of node Buffers + + + + +## [2.0.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v2.0.0...v2.0.1) (2020-07-20) + + +### Bug Fixes + +* pass peer id to onPeerConnect ([#234](https://github.com/ipfs/js-ipfs-bitswap/issues/234)) ([bf3bf0c](https://github.com/ipfs/js-ipfs-bitswap/commit/bf3bf0c)), closes [ipfs/js-ipfs#3182](https://github.com/ipfs/js-ipfs/issues/3182) + + + + +# [2.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v1.0.0...v2.0.0) (2020-06-05) + + +### Features + +* use libp2p 0.28.x ([#217](https://github.com/ipfs/js-ipfs-bitswap/issues/217)) ([c4ede4d](https://github.com/ipfs/js-ipfs-bitswap/commit/c4ede4d)) + + +### BREAKING CHANGES + +* Requires `libp2p@0.28.x` or above + +Co-authored-by: Jacob Heun + + + + +# [1.0.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.29.2...v1.0.0) (2020-05-27) + + +### Bug Fixes + +* do not rebroadcast want list ([#225](https://github.com/ipfs/js-ipfs-bitswap/issues/225)) ([313ae3b](https://github.com/ipfs/js-ipfs-bitswap/commit/313ae3b)), closes [#160](https://github.com/ipfs/js-ipfs-bitswap/issues/160) +* race condition when requesting the same block twice ([#214](https://github.com/ipfs/js-ipfs-bitswap/issues/214)) ([78ce032](https://github.com/ipfs/js-ipfs-bitswap/commit/78ce032)) + + +### Performance Improvements + +* decrease wantlist send debounce time ([#224](https://github.com/ipfs/js-ipfs-bitswap/issues/224)) ([46490f5](https://github.com/ipfs/js-ipfs-bitswap/commit/46490f5)) + + + + +## [0.29.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.29.1...v0.29.2) (2020-05-07) + + +### Bug Fixes + +* re-sort queue after adding tasks ([#221](https://github.com/ipfs/js-ipfs-bitswap/issues/221)) ([1a5ed4a](https://github.com/ipfs/js-ipfs-bitswap/commit/1a5ed4a)), closes [ipfs/js-ipfs#2992](https://github.com/ipfs/js-ipfs/issues/2992) +* survive bad network requests ([#222](https://github.com/ipfs/js-ipfs-bitswap/issues/222)) ([2fc7023](https://github.com/ipfs/js-ipfs-bitswap/commit/2fc7023)), closes [#221](https://github.com/ipfs/js-ipfs-bitswap/issues/221) +* **ci:** add empty commit to fix lint checks on master ([7872a19](https://github.com/ipfs/js-ipfs-bitswap/commit/7872a19)) + + + + +## [0.29.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.29.0...v0.29.1) (2020-04-27) + + +### Bug Fixes + +* really remove node globals ([#219](https://github.com/ipfs/js-ipfs-bitswap/issues/219)) ([120d1c7](https://github.com/ipfs/js-ipfs-bitswap/commit/120d1c7)) + + + + +# [0.29.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.28.0...v0.29.0) (2020-04-23) + + +### Bug Fixes + +* use ipld-block and remove node globals ([#218](https://github.com/ipfs/js-ipfs-bitswap/issues/218)) ([6b4dc32](https://github.com/ipfs/js-ipfs-bitswap/commit/6b4dc32)) + + +### BREAKING CHANGES + +* swaps ipfs-block with ipld-block + +related to https://github.com/ipfs/js-ipfs/issues/2924 + + + + +# [0.28.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.27.1...v0.28.0) (2020-04-09) + + + + +## [0.27.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.27.0...v0.27.1) (2020-02-10) + + +### Bug Fixes + +* await result of receiving blocks ([#213](https://github.com/ipfs/js-ipfs-bitswap/issues/213)) ([dae48dd](https://github.com/ipfs/js-ipfs-bitswap/commit/dae48dd)) + + + + +# [0.27.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.26.2...v0.27.0) (2020-01-28) + + + + +## [0.26.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.26.1...v0.26.2) (2019-12-22) + + +### Bug Fixes + +* use multicodec correctly ([#209](https://github.com/ipfs/js-ipfs-bitswap/issues/209)) ([579ddb5](https://github.com/ipfs/js-ipfs-bitswap/commit/579ddb5)) + + + + +## [0.26.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.26.0...v0.26.1) (2019-12-11) + + +### Bug Fixes + +* reduce size ([#203](https://github.com/ipfs/js-ipfs-bitswap/issues/203)) ([9f818b4](https://github.com/ipfs/js-ipfs-bitswap/commit/9f818b4)) + + + + +# [0.26.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.25.1...v0.26.0) (2019-09-24) + + +### Code Refactoring + +* callbacks -> async / await ([#202](https://github.com/ipfs/js-ipfs-bitswap/issues/202)) ([accf53b](https://github.com/ipfs/js-ipfs-bitswap/commit/accf53b)) + + +### BREAKING CHANGES + +* All places in the API that used callbacks are now replaced with async/await + +* feat: make `get()` a generator + +* make `getMany()` AsyncIterable + +* feat: make `put()` a generator + +* make `putMany()` AsyncIterable + +* remove check in `_findAndConnect()` + +* feat: make `start()` and `stop()` async/await + +* refactor: make `connectTo()` async/await + +* refactor: make `findProviders()` and `findAndConnect()` async/await + +* refactor: cb => async + +* refactor: async/await + +* chore: update travis + +* refactor: update benchmark tests and allow streaming to putMany + +* chore: address pr comments + +* chore: remove callback hell eslint disables + +* chore: wrap list of tasks in promise.all + +* chore: callbackify methods inside pull stream + +* chore: accept PR suggestions + +* chore: fix typo + + + + +## [0.25.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.25.0...v0.25.1) (2019-06-26) + + +### Bug Fixes + +* use consistent encoding for cid comparison ([c8cee6a](https://github.com/ipfs/js-ipfs-bitswap/commit/c8cee6a)) + + +### BREAKING CHANGES + +* Emitted events have different bytes + +The emitted events contain the stringified version of the CID, as we +change it to the base encoding the CID has, those bytes may be different +to previous versions of this module. + +Though this shouldn't have any impact on any other modules as the +events are only used internally. + + + + +# [0.25.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.24.1...v0.25.0) (2019-06-12) + + +### Bug Fixes + +* base encode CIDs before logging or emitting them ([704de22](https://github.com/ipfs/js-ipfs-bitswap/commit/704de22)) + + +### BREAKING CHANGES + +* Emitted events have different bytes + +The emitted events contain the stringified version of the CID, as we +change it to the base encoding the CID has, those bytes may be different +to previous versions of this module. + +Though this shouldn't have any impact on any other modules as the +events are only used internally. + + + + +## [0.24.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.24.0...v0.24.1) (2019-05-30) + + +### Bug Fixes + +* ignore unwanted blocks ([#194](https://github.com/ipfs/js-ipfs-bitswap/issues/194)) ([e8d722c](https://github.com/ipfs/js-ipfs-bitswap/commit/e8d722c)) + + + + +# [0.24.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.23.0...v0.24.0) (2019-05-09) + + +### Chores + +* update cids dependency ([0779160](https://github.com/ipfs/js-ipfs-bitswap/commit/0779160)) + + +### BREAKING CHANGES + +* v1 CIDs created by this module now default to base32 encoding when stringified + +refs: https://github.com/ipfs/js-ipfs/issues/1995 + +License: MIT +Signed-off-by: Alan Shaw + + + + +# [0.23.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.22.0...v0.23.0) (2019-03-16) + + + + +# [0.22.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.21.2...v0.22.0) (2019-01-08) + + +### Bug Fixes + +* reduce bundle size ([d8f8040](https://github.com/ipfs/js-ipfs-bitswap/commit/d8f8040)) + + +### BREAKING CHANGES + +* change from big.js to bignumber.js + +The impact of this change is only on the `snapshot` field of +the stats, as those values are represented as Big Numbers. + + + + +## [0.21.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.21.1...v0.21.2) (2019-01-08) + + +### Bug Fixes + +* avoid sync callbacks in async code ([ddfdd71](https://github.com/ipfs/js-ipfs-bitswap/commit/ddfdd71)) +* ensure callback is called ([c27318f](https://github.com/ipfs/js-ipfs-bitswap/commit/c27318f)) + + + + +## [0.21.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.21.0...v0.21.1) (2018-12-06) + + +### Features + +* send max providers to findProviders request ([31493dc](https://github.com/ipfs/js-ipfs-bitswap/commit/31493dc)) + + + + +# [0.21.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.20.3...v0.21.0) (2018-10-26) + + +### Features + +* change bitswapLedgerForPeer output format ([c68a0c8](https://github.com/ipfs/js-ipfs-bitswap/commit/c68a0c8)) + + + + +## [0.20.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.20.2...v0.20.3) (2018-07-03) + + + + +## [0.20.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.20.0...v0.20.2) (2018-06-18) + + +### Bug Fixes + +* ipfs/js-ipfs[#1292](https://github.com/ipfs/js-ipfs-bitswap/issues/1292) - Catch invalid CIDs and return the error via callback ([#170](https://github.com/ipfs/js-ipfs-bitswap/issues/170)) ([51f5ce0](https://github.com/ipfs/js-ipfs-bitswap/commit/51f5ce0)) +* reset batch size counter ([739ad0d](https://github.com/ipfs/js-ipfs-bitswap/commit/739ad0d)) + + +### Features + +* add bitswap.ledgerForPeer ([871d0d2](https://github.com/ipfs/js-ipfs-bitswap/commit/871d0d2)) +* add ledger.debtRatio() ([e602810](https://github.com/ipfs/js-ipfs-bitswap/commit/e602810)) + + + + +## [0.20.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.20.0...v0.20.1) (2018-05-28) + + +### Bug Fixes + +* ipfs/js-ipfs[#1292](https://github.com/ipfs/js-ipfs-bitswap/issues/1292) - Catch invalid CIDs and return the error via callback ([#170](https://github.com/ipfs/js-ipfs-bitswap/issues/170)) ([51f5ce0](https://github.com/ipfs/js-ipfs-bitswap/commit/51f5ce0)) +* reset batch size counter ([739ad0d](https://github.com/ipfs/js-ipfs-bitswap/commit/739ad0d)) + + + + +# [0.20.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.19.0...v0.20.0) (2018-04-10) + + + + +# [0.19.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.18.1...v0.19.0) (2018-02-14) + + +### Features + +* update network calls to use dialProtocol instead ([b669aac](https://github.com/ipfs/js-ipfs-bitswap/commit/b669aac)) + + + + +## [0.18.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.18.0...v0.18.1) (2018-02-06) + + +### Bug Fixes + +* getMany: ensuring we set the want list ([#162](https://github.com/ipfs/js-ipfs-bitswap/issues/162)) ([8e91def](https://github.com/ipfs/js-ipfs-bitswap/commit/8e91def)) + + +### Features + +* added getMany performance tests ([#164](https://github.com/ipfs/js-ipfs-bitswap/issues/164)) ([b349085](https://github.com/ipfs/js-ipfs-bitswap/commit/b349085)) +* per-peer stats ([#166](https://github.com/ipfs/js-ipfs-bitswap/issues/166)) ([ff978d0](https://github.com/ipfs/js-ipfs-bitswap/commit/ff978d0)) + + + + +# [0.18.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.4...v0.18.0) (2017-12-15) + + +### Features + +* stats improvements ([#158](https://github.com/ipfs/js-ipfs-bitswap/issues/158)) ([17e15d0](https://github.com/ipfs/js-ipfs-bitswap/commit/17e15d0)) + + + + +## [0.17.4](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.3...v0.17.4) (2017-11-10) + + +### Features + +* windows interop ([#154](https://github.com/ipfs/js-ipfs-bitswap/issues/154)) ([a8b1e07](https://github.com/ipfs/js-ipfs-bitswap/commit/a8b1e07)) + + + + +## [0.17.3](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.2...v0.17.3) (2017-11-08) + + +### Bug Fixes + +* add missing multicodec dependency ([#155](https://github.com/ipfs/js-ipfs-bitswap/issues/155)) ([751d436](https://github.com/ipfs/js-ipfs-bitswap/commit/751d436)) + + + + +## [0.17.2](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.1...v0.17.2) (2017-09-07) + + + + +## [0.17.1](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.17.0...v0.17.1) (2017-09-07) + + +### Features + +* replace protocol-buffers with protons ([#149](https://github.com/ipfs/js-ipfs-bitswap/issues/149)) ([ca8fa72](https://github.com/ipfs/js-ipfs-bitswap/commit/ca8fa72)) + + + + +# [0.17.0](https://github.com/ipfs/js-ipfs-bitswap/compare/v0.16.1...v0.17.0) (2017-09-03) diff --git a/packages/bitswap/LICENSE b/packages/bitswap/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/bitswap/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/bitswap/LICENSE-APACHE b/packages/bitswap/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/bitswap/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/bitswap/LICENSE-MIT b/packages/bitswap/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/bitswap/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/bitswap/README.md b/packages/bitswap/README.md new file mode 100644 index 00000000..163a5cec --- /dev/null +++ b/packages/bitswap/README.md @@ -0,0 +1,64 @@ +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain) + +> JavaScript implementation of the Bitswap data exchange protocol used by Helia + +# About + + + +This module implements the [Bitswap protocol](https://docs.ipfs.tech/concepts/bitswap/) in TypeScript. + +It supersedes the older [ipfs-bitswap](https://www.npmjs.com/package/ipfs-bitswap) module with the aim of being smaller, faster, better integrated with libp2p/helia, having fewer dependencies and using standard JavaScript instead of Node.js APIs. + +# Install + +```console +$ npm i @helia/bitswap +``` + +## Browser ` +``` + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/bitswap/package.json b/packages/bitswap/package.json new file mode 100644 index 00000000..aa352129 --- /dev/null +++ b/packages/bitswap/package.json @@ -0,0 +1,200 @@ +{ + "name": "@helia/bitswap", + "version": "0.0.0", + "description": "JavaScript implementation of the Bitswap data exchange protocol used by Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia/tree/main/packages/bitswap#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia/issues" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "keywords": [ + "exchange", + "ipfs", + "libp2p", + "p2p" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "project": true, + "sourceType": "module" + }, + "ignorePatterns": [ + "scripts/*", + "*.test-d.ts" + ] + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "build": "aegir build", + "release": "aegir release", + "test": "aegir test", + "test:node": "aegir test -t node --cov", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:electron-main": "aegir test -t electron-main", + "dep-check": "aegir dep-check -i protons -i events", + "generate": "protons ./src/pb/message.proto", + "docs": "aegir docs" + }, + "dependencies": { + "@helia/interface": "^4.0.0", + "@libp2p/interface": "^1.1.2", + "@libp2p/logger": "^4.0.5", + "@libp2p/peer-collections": "^5.1.6", + "@libp2p/utils": "^5.2.3", + "@multiformats/multiaddr": "^12.1.14", + "@multiformats/multiaddr-matcher": "^1.1.2", + "any-signal": "^4.1.1", + "debug": "^4.3.4", + "interface-blockstore": "^5.2.9", + "interface-store": "^5.1.7", + "it-all": "^3.0.4", + "it-drain": "^3.0.5", + "it-filter": "^3.0.4", + "it-length-prefixed": "^9.0.0", + "it-length-prefixed-stream": "^1.1.6", + "it-map": "^3.0.5", + "it-merge": "^3.0.3", + "it-pipe": "^3.0.1", + "it-take": "^3.0.1", + "multiformats": "^13.0.1", + "p-defer": "^4.0.0", + "progress-events": "^1.0.0", + "protons-runtime": "^5.0.0", + "race-event": "^1.2.0", + "uint8-varint": "^2.0.3", + "uint8arraylist": "^2.4.3", + "uint8arrays": "^5.0.1" + }, + "devDependencies": { + "@libp2p/interface-compliance-tests": "^5.1.3", + "@libp2p/peer-id-factory": "^4.0.5", + "@types/sinon": "^17.0.3", + "aegir": "^42.2.2", + "blockstore-core": "^4.3.10", + "delay": "^6.0.0", + "it-pair": "^2.0.6", + "it-protobuf-stream": "^1.1.2", + "p-event": "^6.0.0", + "p-retry": "^6.2.0", + "p-wait-for": "^5.0.2", + "protons": "^7.0.2", + "sinon": "^17.0.1", + "sinon-ts": "^2.0.0" + }, + "browser": { + "dist/test/utils/create-libp2p-node.js": false + }, + "sideEffects": false +} diff --git a/packages/bitswap/src/bitswap.ts b/packages/bitswap/src/bitswap.ts new file mode 100644 index 00000000..0c70ed71 --- /dev/null +++ b/packages/bitswap/src/bitswap.ts @@ -0,0 +1,152 @@ +/* eslint-disable no-loop-func */ +import { DEFAULT_SESSION_MAX_PROVIDERS, DEFAULT_SESSION_MIN_PROVIDERS, DEFAULT_SESSION_PROVIDER_QUERY_CONCURRENCY } from '@helia/interface' +import { setMaxListeners } from '@libp2p/interface' +import { anySignal } from 'any-signal' +import { Network } from './network.js' +import { PeerWantLists } from './peer-want-lists/index.js' +import { createBitswapSession } from './session.js' +import { Stats } from './stats.js' +import { WantList } from './want-list.js' +import type { BitswapOptions, Bitswap as BitswapInterface, BitswapWantProgressEvents, BitswapNotifyProgressEvents, BitswapSession, WantListEntry, BitswapComponents, CreateBitswapSessionOptions } from './index.js' +import type { ComponentLogger, PeerId } from '@libp2p/interface' +import type { Logger } from '@libp2p/logger' +import type { AbortOptions } from '@multiformats/multiaddr' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' +import type { ProgressOptions } from 'progress-events' + +export interface WantOptions extends AbortOptions, ProgressOptions { + /** + * When searching the routing for providers, stop searching after finding this + * many providers. + * + * @default 3 + */ + maxProviders?: number +} + +/** + * JavaScript implementation of the Bitswap 'data exchange' protocol + * used by IPFS. + */ +export class Bitswap implements BitswapInterface { + private readonly log: Logger + private readonly logger: ComponentLogger + public readonly stats: Stats + public network: Network + public blockstore: Blockstore + public peerWantLists: PeerWantLists + public wantList: WantList + + constructor (components: BitswapComponents, init: BitswapOptions = {}) { + this.logger = components.logger + this.log = components.logger.forComponent('helia:bitswap') + this.blockstore = components.blockstore + + // report stats to libp2p metrics + this.stats = new Stats(components) + + // the network delivers messages + this.network = new Network(components, init) + + // handle which blocks we send to peers + this.peerWantLists = new PeerWantLists({ + ...components, + network: this.network + }, init) + + // handle which blocks we ask peers for + this.wantList = new WantList({ + ...components, + network: this.network + }, init) + } + + async createSession (root: CID, options?: CreateBitswapSessionOptions): Promise { + const minProviders = options?.minProviders ?? DEFAULT_SESSION_MIN_PROVIDERS + const maxProviders = options?.maxProviders ?? DEFAULT_SESSION_MAX_PROVIDERS + + return createBitswapSession({ + wantList: this.wantList, + network: this.network, + logger: this.logger + }, { + root, + queryConcurrency: options?.providerQueryConcurrency ?? DEFAULT_SESSION_PROVIDER_QUERY_CONCURRENCY, + minProviders, + maxProviders, + connectedPeers: options?.queryConnectedPeers !== false ? [...this.wantList.peers.keys()] : [], + signal: options?.signal + }) + } + + async want (cid: CID, options: WantOptions = {}): Promise { + const controller = new AbortController() + setMaxListeners(Infinity, controller.signal) + const signal = anySignal([controller.signal, options.signal]) + + // find providers and connect to them + this.network.findAndConnect(cid, { + ...options, + signal + }) + .catch(err => { + // if the controller was aborted we found the block already so ignore + // the error + if (!controller.signal.aborted) { + this.log.error('error during finding and connect for cid %c', cid, err) + } + }) + + try { + const result = await this.wantList.wantBlock(cid, { + ...options, + signal + }) + + return result.block + } finally { + // since we have the block we can now abort any outstanding attempts to + // find providers for it + controller.abort() + signal.clear() + } + } + + /** + * Sends notifications about the arrival of a block + */ + async notify (cid: CID, block: Uint8Array, options: ProgressOptions & AbortOptions = {}): Promise { + await this.peerWantLists.receivedBlock(cid, options) + } + + getWantlist (): WantListEntry[] { + return [...this.wantList.wants.values()] + .filter(entry => !entry.cancel) + .map(entry => ({ + cid: entry.cid, + priority: entry.priority, + wantType: entry.wantType + })) + } + + getPeerWantlist (peer: PeerId): WantListEntry[] | undefined { + return this.peerWantLists.wantListForPeer(peer) + } + + /** + * Start the bitswap node + */ + async start (): Promise { + this.wantList.start() + await this.network.start() + } + + /** + * Stop the bitswap node + */ + async stop (): Promise { + this.wantList.stop() + await this.network.stop() + } +} diff --git a/packages/bitswap/src/constants.ts b/packages/bitswap/src/constants.ts new file mode 100644 index 00000000..8657799b --- /dev/null +++ b/packages/bitswap/src/constants.ts @@ -0,0 +1,11 @@ +export const BITSWAP_120 = '/ipfs/bitswap/1.2.0' +export const DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK = 1024 +export const DEFAULT_MAX_INBOUND_STREAMS = 1024 +export const DEFAULT_MAX_OUTBOUND_STREAMS = 1024 +export const DEFAULT_MESSAGE_RECEIVE_TIMEOUT = 5000 +export const DEFAULT_MESSAGE_SEND_DELAY = 10 +export const DEFAULT_MESSAGE_SEND_TIMEOUT = 5000 +export const DEFAULT_MESSAGE_SEND_CONCURRENCY = 50 +export const DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS = false +export const DEFAULT_SESSION_ROOT_PRIORITY = 1 +export const DEFAULT_MAX_PROVIDERS_PER_REQUEST = 3 diff --git a/packages/bitswap/src/index.ts b/packages/bitswap/src/index.ts new file mode 100644 index 00000000..a2ff51c9 --- /dev/null +++ b/packages/bitswap/src/index.ts @@ -0,0 +1,215 @@ +/** + * @packageDocumentation + * + * This module implements the [Bitswap protocol](https://docs.ipfs.tech/concepts/bitswap/) in TypeScript. + * + * It supersedes the older [ipfs-bitswap](https://www.npmjs.com/package/ipfs-bitswap) module with the aim of being smaller, faster, better integrated with libp2p/helia, having fewer dependencies and using standard JavaScript instead of Node.js APIs. + */ + +import { Bitswap as BitswapClass } from './bitswap.js' +import type { BitswapNetworkNotifyProgressEvents, BitswapNetworkWantProgressEvents } from './network.js' +import type { WantType } from './pb/message.js' +import type { CreateSessionOptions } from '@helia/interface' +import type { Routing } from '@helia/interface/routing' +import type { Libp2p, AbortOptions, Startable, ComponentLogger, Metrics, PeerId } from '@libp2p/interface' +import type { PeerSet } from '@libp2p/peer-collections' +import type { Blockstore } from 'interface-blockstore' +import type { CID } from 'multiformats/cid' +import type { MultihashHasher } from 'multiformats/hashes/interface' +import type { ProgressEvent, ProgressOptions } from 'progress-events' + +export type BitswapWantProgressEvents = + BitswapWantBlockProgressEvents + +export type BitswapNotifyProgressEvents = + BitswapNetworkNotifyProgressEvents + +export type BitswapWantBlockProgressEvents = + ProgressEvent<'bitswap:want-block:unwant', CID> | + ProgressEvent<'bitswap:want-block:block', CID> | + BitswapNetworkWantProgressEvents + +/** + * A bitswap session is a network overlay consisting of peers that all have the + * first block in a file. Subsequent requests will only go to these peers. + */ +export interface BitswapSession { + /** + * The peers in this session + */ + peers: PeerSet + + /** + * Fetch an additional CID from this DAG + */ + want(cid: CID, options?: AbortOptions & ProgressOptions): Promise +} + +export interface WantListEntry { + cid: CID + priority: number + wantType: WantType +} + +export interface CreateBitswapSessionOptions extends CreateSessionOptions { + /** + * If true, query connected peers before searching for providers via + * Helia routers + * + * @default true + */ + queryConnectedPeers?: boolean + + /** + * If true, search for providers via Helia routers to query for the root CID + * + * @default true + */ + queryRoutingPeers?: boolean + + /** + * The priority to use when querying availability of the root CID + * + * @default 1 + */ + priority?: number +} + +export interface Bitswap extends Startable { + /** + * Returns the current state of the wantlist + */ + getWantlist(): WantListEntry[] + + /** + * Returns the current state of the wantlist for a peer, if it is being + * tracked + */ + getPeerWantlist(peerId: PeerId): WantListEntry[] | undefined + + /** + * Notify bitswap that a new block is available + */ + notify(cid: CID, block: Uint8Array, options?: ProgressOptions): Promise + + /** + * Start a session to retrieve a file from the network + */ + want(cid: CID, options?: AbortOptions & ProgressOptions): Promise + + /** + * Start a session to retrieve a file from the network + */ + createSession(root: CID, options?: AbortOptions & ProgressOptions): Promise +} + +export interface MultihashHasherLoader { + getHasher(codeOrName: number | string): Promise +} + +export interface BitswapComponents { + routing: Routing + blockstore: Blockstore + logger: ComponentLogger + libp2p: Libp2p + metrics?: Metrics +} + +export interface BitswapOptions { + /** + * This is the maximum number of concurrent inbound bitswap streams that are + * allowed + * + * @default 32 + */ + maxInboundStreams?: number + + /** + * This is the maximum number of concurrent outbound bitswap streams that are + * allowed + * + * @default 128 + */ + maxOutboundStreams?: number + + /** + * An incoming stream must resolve within this number of seconds + * + * @default 30000 + */ + incomingStreamTimeout?: number + + /** + * Whether to run on transient (e.g. time/data limited) connections + * + * @default false + */ + runOnTransientConnections?: boolean + + /** + * Enables loading esoteric hash functions + */ + hashLoader?: MultihashHasherLoader + + /** + * The protocol that we speak + * + * @default '/ipfs/bitswap/1.2.0' + */ + protocol?: string + + /** + * When a new peer connects, sending our WantList should complete within this + * many ms + * + * @default 5000 + */ + messageSendTimeout?: number + + /** + * When sending want list updates to peers, how many messages to send at once + * + * @default 50 + */ + messageSendConcurrency?: number + + /** + * When sending blocks to peers, how many messages to send at once + * + * @default 50 + */ + sendBlocksConcurrency?: number + + /** + * When sending blocks to peers, timeout after this many milliseconds. + * This is useful for preventing slow/large peer-connections from consuming + * your bandwidth/streams. + * + * @default 10000 + */ + sendBlocksTimeout?: number + + /** + * When a block is added to the blockstore and we are about to send that block + * to peers who have it in their wantlist, wait this many milliseconds before + * queueing the send job in case more blocks are added that they want + * + * @default 10 + */ + sendBlocksDebounce?: number + + /** + * If the client sends a want-have, and we have the corresponding block, we + * check the size of the block and if it's small enough we send the block + * itself, rather than sending a HAVE. + * + * This defines the maximum size up to which we replace a HAVE with a block. + * + * @default 1024 + */ + maxSizeReplaceHasWithBlock?: number +} + +export const createBitswap = (components: BitswapComponents, options: BitswapOptions = {}): Bitswap => { + return new BitswapClass(components, options) +} diff --git a/packages/bitswap/src/network.ts b/packages/bitswap/src/network.ts new file mode 100644 index 00000000..17dc36fb --- /dev/null +++ b/packages/bitswap/src/network.ts @@ -0,0 +1,506 @@ +import { CodeError, TypedEventEmitter, setMaxListeners } from '@libp2p/interface' +import { PeerQueue, type PeerQueueJobOptions } from '@libp2p/utils/peer-queue' +import { Circuit } from '@multiformats/multiaddr-matcher' +import { anySignal } from 'any-signal' +import debug from 'debug' +import drain from 'it-drain' +import * as lp from 'it-length-prefixed' +import { lpStream } from 'it-length-prefixed-stream' +import map from 'it-map' +import { pipe } from 'it-pipe' +import take from 'it-take' +import { base64 } from 'multiformats/bases/base64' +import { CID } from 'multiformats/cid' +import { CustomProgressEvent } from 'progress-events' +import { raceEvent } from 'race-event' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { BITSWAP_120, DEFAULT_MAX_INBOUND_STREAMS, DEFAULT_MAX_OUTBOUND_STREAMS, DEFAULT_MAX_PROVIDERS_PER_REQUEST, DEFAULT_MESSAGE_RECEIVE_TIMEOUT, DEFAULT_MESSAGE_SEND_TIMEOUT, DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS } from './constants.js' +import { BitswapMessage } from './pb/message.js' +import type { WantOptions } from './bitswap.js' +import type { MultihashHasherLoader } from './index.js' +import type { Block, BlockPresence, WantlistEntry } from './pb/message.js' +import type { Provider, Routing } from '@helia/interface/routing' +import type { Libp2p, AbortOptions, Connection, PeerId, IncomingStreamData, Topology, MetricGroup, ComponentLogger, Metrics, IdentifyResult } from '@libp2p/interface' +import type { Logger } from '@libp2p/logger' +import type { ProgressEvent, ProgressOptions } from 'progress-events' + +// Add a formatter for a bitswap message +debug.formatters.B = (b?: BitswapMessage): string => { + if (b == null) { + return 'undefined' + } + + return JSON.stringify({ + blocks: b.blocks?.map(b => ({ + data: `${uint8ArrayToString(b.data, 'base64').substring(0, 10)}...`, + prefix: uint8ArrayToString(b.prefix, 'base64') + })), + blockPresences: b.blockPresences?.map(p => ({ + ...p, + cid: CID.decode(p.cid).toString() + })), + wantlist: b.wantlist == null + ? undefined + : { + full: b.wantlist.full, + entries: b.wantlist.entries.map(e => ({ + ...e, + cid: CID.decode(e.cid).toString() + })) + } + }, null, 2) +} + +export type BitswapNetworkProgressEvents = + ProgressEvent<'bitswap:network:dial', PeerId> + +export type BitswapNetworkWantProgressEvents = + ProgressEvent<'bitswap:network:send-wantlist', PeerId> | + ProgressEvent<'bitswap:network:send-wantlist:error', { peer: PeerId, error: Error }> | + ProgressEvent<'bitswap:network:find-providers', CID> | + BitswapNetworkProgressEvents + +export type BitswapNetworkNotifyProgressEvents = + BitswapNetworkProgressEvents | + ProgressEvent<'bitswap:network:send-block', PeerId> + +export interface NetworkInit { + hashLoader?: MultihashHasherLoader + maxInboundStreams?: number + maxOutboundStreams?: number + messageReceiveTimeout?: number + messageSendTimeout?: number + messageSendConcurrency?: number + protocols?: string[] + runOnTransientConnections?: boolean +} + +export interface NetworkComponents { + routing: Routing + logger: ComponentLogger + libp2p: Libp2p + metrics?: Metrics +} + +export interface BitswapMessageEventDetail { + peer: PeerId + message: BitswapMessage +} + +export interface NetworkEvents { + 'bitswap:message': CustomEvent<{ peer: PeerId, message: BitswapMessage }> + 'peer:connected': CustomEvent + 'peer:disconnected': CustomEvent +} + +interface SendMessageJobOptions extends AbortOptions, ProgressOptions, PeerQueueJobOptions { + message: BitswapMessage +} + +export class Network extends TypedEventEmitter { + private readonly log: Logger + private readonly libp2p: Libp2p + private readonly routing: Routing + private readonly protocols: string[] + private running: boolean + private readonly maxInboundStreams: number + private readonly maxOutboundStreams: number + private readonly messageReceiveTimeout: number + private registrarIds: string[] + private readonly metrics?: { blocksSent: MetricGroup, dataSent: MetricGroup } + private readonly sendQueue: PeerQueue + private readonly messageSendTimeout: number + private readonly runOnTransientConnections: boolean + + constructor (components: NetworkComponents, init: NetworkInit = {}) { + super() + + this.log = components.logger.forComponent('helia:bitswap:network') + this.libp2p = components.libp2p + this.routing = components.routing + this.protocols = init.protocols ?? [BITSWAP_120] + this.registrarIds = [] + this.running = false + + // bind event listeners + this._onStream = this._onStream.bind(this) + this.maxInboundStreams = init.maxInboundStreams ?? DEFAULT_MAX_INBOUND_STREAMS + this.maxOutboundStreams = init.maxOutboundStreams ?? DEFAULT_MAX_OUTBOUND_STREAMS + this.messageReceiveTimeout = init.messageReceiveTimeout ?? DEFAULT_MESSAGE_RECEIVE_TIMEOUT + this.messageSendTimeout = init.messageSendTimeout ?? DEFAULT_MESSAGE_SEND_TIMEOUT + this.runOnTransientConnections = init.runOnTransientConnections ?? DEFAULT_RUN_ON_TRANSIENT_CONNECTIONS + + if (components.metrics != null) { + this.metrics = { + blocksSent: components.metrics?.registerMetricGroup('ipfs_bitswap_sent_blocks'), + dataSent: components.metrics?.registerMetricGroup('ipfs_bitswap_sent_data_bytes') + } + } + + this.sendQueue = new PeerQueue({ + concurrency: init.messageSendConcurrency, + metrics: components.metrics, + metricName: 'ipfs_bitswap_message_send_queue' + }) + this.sendQueue.addEventListener('error', (evt) => { + this.log.error('error sending wantlist to peer', evt.detail) + }) + } + + async start (): Promise { + if (this.running) { + return + } + + this.running = true + + await this.libp2p.handle(this.protocols, this._onStream, { + maxInboundStreams: this.maxInboundStreams, + maxOutboundStreams: this.maxOutboundStreams, + runOnTransientConnection: this.runOnTransientConnections + }) + + // register protocol with topology + const topology: Topology = { + onConnect: (peerId: PeerId) => { + this.safeDispatchEvent('peer:connected', { + detail: peerId + }) + }, + onDisconnect: (peerId: PeerId) => { + this.safeDispatchEvent('peer:disconnected', { + detail: peerId + }) + } + } + + this.registrarIds = [] + + for (const protocol of this.protocols) { + this.registrarIds.push(await this.libp2p.register(protocol, topology)) + } + + // All existing connections are like new ones for us + this.libp2p.getConnections().forEach(conn => { + this.safeDispatchEvent('peer:connected', { + detail: conn.remotePeer + }) + }) + } + + async stop (): Promise { + this.running = false + + // Unhandle both, libp2p doesn't care if it's not already handled + await this.libp2p.unhandle(this.protocols) + + // unregister protocol and handlers + if (this.registrarIds != null) { + for (const id of this.registrarIds) { + this.libp2p.unregister(id) + } + + this.registrarIds = [] + } + } + + /** + * Handles incoming bitswap messages + */ + _onStream (info: IncomingStreamData): void { + if (!this.running) { + return + } + + const { stream, connection } = info + + Promise.resolve().then(async () => { + this.log('incoming new bitswap %s stream from %p', stream.protocol, connection.remotePeer) + const abortListener = (): void => { + stream.abort(new CodeError('Incoming Bitswap stream timed out', 'ERR_TIMEOUT')) + } + + let signal = AbortSignal.timeout(this.messageReceiveTimeout) + setMaxListeners(Infinity, signal) + signal.addEventListener('abort', abortListener) + + await pipe( + stream, + (source) => lp.decode(source), + async (source) => { + for await (const data of source) { + try { + const message = BitswapMessage.decode(data) + this.log('incoming new bitswap %s message from %p %B', stream.protocol, connection.remotePeer, message) + + this.safeDispatchEvent('bitswap:message', { + detail: { + peer: connection.remotePeer, + message + } + }) + + // we have received some data so reset the timeout controller + signal.removeEventListener('abort', abortListener) + signal = AbortSignal.timeout(this.messageReceiveTimeout) + setMaxListeners(Infinity, signal) + signal.addEventListener('abort', abortListener) + } catch (err: any) { + this.log.error('error reading incoming bitswap message from %p', connection.remotePeer, err) + stream.abort(err) + break + } + } + } + ) + }) + .catch(err => { + this.log.error('error handling incoming stream from %p', connection.remotePeer, err) + stream.abort(err) + }) + } + + /** + * Find bitswap providers for a given `cid`. + */ + async * findProviders (cid: CID, options?: AbortOptions & ProgressOptions): AsyncIterable { + options?.onProgress?.(new CustomProgressEvent('bitswap:network:find-providers', cid)) + + for await (const provider of this.routing.findProviders(cid, options)) { + // unless we explicitly run on transient connections, skip peers that only + // have circuit relay addresses as bitswap won't run over them + if (!this.runOnTransientConnections) { + let hasDirectAddress = false + + for (let ma of provider.multiaddrs) { + if (ma.getPeerId() == null) { + ma = ma.encapsulate(`/p2p/${provider.id}`) + } + + if (!Circuit.exactMatch(ma)) { + hasDirectAddress = true + break + } + } + + if (!hasDirectAddress) { + continue + } + } + + // ignore non-bitswap providers + if (provider.protocols?.includes('transport-bitswap') === false) { + continue + } + + yield provider + } + } + + /** + * Find the providers of a given `cid` and connect to them. + */ + async findAndConnect (cid: CID, options?: WantOptions): Promise { + await drain( + take( + map(this.findProviders(cid, options), async provider => this.connectTo(provider.id, options)), + options?.maxProviders ?? DEFAULT_MAX_PROVIDERS_PER_REQUEST + ) + ) + .catch(err => { + this.log.error(err) + }) + } + + /** + * Connect to the given peer + * Send the given msg (instance of Message) to the given peer + */ + async sendMessage (peerId: PeerId, msg: Partial, options?: AbortOptions & ProgressOptions): Promise { + if (!this.running) { + throw new Error('network isn\'t running') + } + + const message: BitswapMessage = { + wantlist: { + full: msg.wantlist?.full ?? false, + entries: msg.wantlist?.entries ?? [] + }, + blocks: msg.blocks ?? [], + blockPresences: msg.blockPresences ?? [], + pendingBytes: msg.pendingBytes ?? 0 + } + + const signal = anySignal([AbortSignal.timeout(this.messageSendTimeout), options?.signal]) + setMaxListeners(Infinity, signal) + + try { + const existingJob = this.sendQueue.find(peerId) + + if (existingJob?.status === 'queued') { + // merge messages instead of adding new job + existingJob.options.message = mergeMessages(existingJob.options.message, message) + + await existingJob.join({ + signal + }) + + return + } + + await this.sendQueue.add(async (options) => { + const message = options?.message + + if (message == null) { + throw new CodeError('No message to send', 'ERR_NO_MESSAGE') + } + + this.log('sendMessage to %p %B', peerId, message) + + options?.onProgress?.(new CustomProgressEvent('bitswap:network:send-wantlist', peerId)) + + const stream = await this.libp2p.dialProtocol(peerId, BITSWAP_120, options) + + try { + const lp = lpStream(stream) + await lp.write(BitswapMessage.encode(message), options) + await lp.unwrap().close(options) + } catch (err: any) { + options?.onProgress?.(new CustomProgressEvent<{ peer: PeerId, error: Error }>('bitswap:network:send-wantlist:error', { peer: peerId, error: err })) + this.log.error('error sending message to %p', peerId, err) + stream.abort(err) + } + + this._updateSentStats(peerId, message.blocks) + }, { + peerId, + signal, + message + }) + } finally { + signal.clear() + } + } + + /** + * Connects to another peer + */ + async connectTo (peer: PeerId, options?: AbortOptions & ProgressOptions): Promise { // eslint-disable-line require-await + if (!this.running) { + throw new CodeError('Network isn\'t running', 'ERR_NOT_STARTED') + } + + options?.onProgress?.(new CustomProgressEvent('bitswap:network:dial', peer)) + + // dial and wait for identify - this is to avoid opening a protocol stream + // that we are not going to use but depends on the remote node running the + // identitfy protocol + const [ + connection + ] = await Promise.all([ + this.libp2p.dial(peer, options), + raceEvent(this.libp2p, 'peer:identify', options?.signal, { + filter: (evt: CustomEvent): boolean => { + if (!evt.detail.peerId.equals(peer)) { + return false + } + + if (evt.detail.protocols.includes(BITSWAP_120)) { + return true + } + + throw new CodeError(`${peer} did not support ${BITSWAP_120}`, 'ERR_BITSWAP_UNSUPPORTED_BY_PEER') + } + }) + ]) + + return connection + } + + _updateSentStats (peerId: PeerId, blocks: Block[] = []): void { + if (this.metrics != null) { + let bytes = 0 + + for (const block of blocks.values()) { + bytes += block.data.byteLength + } + + this.metrics.dataSent.increment({ + global: bytes, + [peerId.toString()]: bytes + }) + this.metrics.blocksSent.increment({ + global: blocks.length, + [peerId.toString()]: blocks.length + }) + } + } +} + +function mergeMessages (messageA: BitswapMessage, messageB: BitswapMessage): BitswapMessage { + const wantListEntries = new Map( + (messageA.wantlist?.entries ?? []).map(entry => ([ + base64.encode(entry.cid), + entry + ])) + ) + + for (const entry of messageB.wantlist?.entries ?? []) { + const key = base64.encode(entry.cid) + const existingEntry = wantListEntries.get(key) + + if (existingEntry != null) { + // take highest priority + if (existingEntry.priority > entry.priority) { + entry.priority = existingEntry.priority + } + + // take later values if passed, otherwise use earlier ones + entry.cancel = entry.cancel ?? existingEntry.cancel + entry.wantType = entry.wantType ?? existingEntry.wantType + entry.sendDontHave = entry.sendDontHave ?? existingEntry.sendDontHave + } + + wantListEntries.set(key, entry) + } + + const blockPresences = new Map( + messageA.blockPresences.map(presence => ([ + base64.encode(presence.cid), + presence + ])) + ) + + for (const blockPresence of messageB.blockPresences) { + const key = base64.encode(blockPresence.cid) + + // override earlier block presence with later one as if duplicated it is + // likely to be more accurate since it is more recent + blockPresences.set(key, blockPresence) + } + + const blocks = new Map( + messageA.blocks.map(block => ([ + base64.encode(block.data), + block + ])) + ) + + for (const block of messageB.blocks) { + const key = base64.encode(block.data) + + blocks.set(key, block) + } + + const output: BitswapMessage = { + wantlist: { + full: messageA.wantlist?.full ?? messageB.wantlist?.full ?? false, + entries: [...wantListEntries.values()] + }, + blockPresences: [...blockPresences.values()], + blocks: [...blocks.values()], + pendingBytes: messageA.pendingBytes + messageB.pendingBytes + } + + return output +} diff --git a/packages/bitswap/src/pb/message.proto b/packages/bitswap/src/pb/message.proto new file mode 100644 index 00000000..bed8d177 --- /dev/null +++ b/packages/bitswap/src/pb/message.proto @@ -0,0 +1,42 @@ +// adapted from https://github.com/ipfs/boxo/blob/main/bitswap/message/pb/message.proto +syntax = "proto3"; + +enum WantType { + WantBlock = 0; // send me the block for the CID + WantHave = 1; // just tell me if you have the block for the CID or send it if it's really small +} + +message WantlistEntry { + bytes cid = 1; // the block cid (cidV0 in bitswap 1.0.0, cidV1 in bitswap 1.1.0) + int32 priority = 2; // the priority (normalized). default to 1 + optional bool cancel = 3; // whether this revokes an entry + optional WantType wantType = 4; // Note: defaults to enum 0, ie Block + optional bool sendDontHave = 5; // Note: defaults to false +} + +message Wantlist { + repeated WantlistEntry entries = 1; // a list of wantlist entries + optional bool full = 2; // whether this is the full wantlist. default to false +} + +message Block { + bytes prefix = 1; // CID prefix (cid version, multicodec and multihash prefix (type + length) + bytes data = 2; +} + +enum BlockPresenceType { + HaveBlock = 0; + DontHaveBlock = 1; +} + +message BlockPresence { + bytes cid = 1; + BlockPresenceType type = 2; +} + +message BitswapMessage { + Wantlist wantlist = 1; + repeated Block blocks = 3; // used to send Blocks in bitswap 1.1.0 + repeated BlockPresence blockPresences = 4; + int32 pendingBytes = 5; +} diff --git a/packages/bitswap/src/pb/message.ts b/packages/bitswap/src/pb/message.ts new file mode 100644 index 00000000..ad7b47ea --- /dev/null +++ b/packages/bitswap/src/pb/message.ts @@ -0,0 +1,450 @@ +/* eslint-disable import/export */ +/* eslint-disable complexity */ +/* eslint-disable @typescript-eslint/no-namespace */ +/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ +/* eslint-disable @typescript-eslint/no-empty-interface */ + +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' +import type { Uint8ArrayList } from 'uint8arraylist' + +export enum WantType { + WantBlock = 'WantBlock', + WantHave = 'WantHave' +} + +enum __WantTypeValues { + WantBlock = 0, + WantHave = 1 +} + +export namespace WantType { + export const codec = (): Codec => { + return enumeration(__WantTypeValues) + } +} +export interface WantlistEntry { + cid: Uint8Array + priority: number + cancel?: boolean + wantType?: WantType + sendDontHave?: boolean +} + +export namespace WantlistEntry { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.cid != null && obj.cid.byteLength > 0)) { + w.uint32(10) + w.bytes(obj.cid) + } + + if ((obj.priority != null && obj.priority !== 0)) { + w.uint32(16) + w.int32(obj.priority) + } + + if (obj.cancel != null) { + w.uint32(24) + w.bool(obj.cancel) + } + + if (obj.wantType != null) { + w.uint32(32) + WantType.codec().encode(obj.wantType, w) + } + + if (obj.sendDontHave != null) { + w.uint32(40) + w.bool(obj.sendDontHave) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + cid: uint8ArrayAlloc(0), + priority: 0 + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.cid = reader.bytes() + break + } + case 2: { + obj.priority = reader.int32() + break + } + case 3: { + obj.cancel = reader.bool() + break + } + case 4: { + obj.wantType = WantType.codec().decode(reader) + break + } + case 5: { + obj.sendDontHave = reader.bool() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, WantlistEntry.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): WantlistEntry => { + return decodeMessage(buf, WantlistEntry.codec()) + } +} + +export interface Wantlist { + entries: WantlistEntry[] + full?: boolean +} + +export namespace Wantlist { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if (obj.entries != null) { + for (const value of obj.entries) { + w.uint32(10) + WantlistEntry.codec().encode(value, w) + } + } + + if (obj.full != null) { + w.uint32(16) + w.bool(obj.full) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + entries: [] + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.entries.push(WantlistEntry.codec().decode(reader, reader.uint32())) + break + } + case 2: { + obj.full = reader.bool() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, Wantlist.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): Wantlist => { + return decodeMessage(buf, Wantlist.codec()) + } +} + +export interface Block { + prefix: Uint8Array + data: Uint8Array +} + +export namespace Block { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.prefix != null && obj.prefix.byteLength > 0)) { + w.uint32(10) + w.bytes(obj.prefix) + } + + if ((obj.data != null && obj.data.byteLength > 0)) { + w.uint32(18) + w.bytes(obj.data) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + prefix: uint8ArrayAlloc(0), + data: uint8ArrayAlloc(0) + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.prefix = reader.bytes() + break + } + case 2: { + obj.data = reader.bytes() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, Block.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): Block => { + return decodeMessage(buf, Block.codec()) + } +} + +export enum BlockPresenceType { + HaveBlock = 'HaveBlock', + DontHaveBlock = 'DontHaveBlock' +} + +enum __BlockPresenceTypeValues { + HaveBlock = 0, + DontHaveBlock = 1 +} + +export namespace BlockPresenceType { + export const codec = (): Codec => { + return enumeration(__BlockPresenceTypeValues) + } +} +export interface BlockPresence { + cid: Uint8Array + type: BlockPresenceType +} + +export namespace BlockPresence { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if ((obj.cid != null && obj.cid.byteLength > 0)) { + w.uint32(10) + w.bytes(obj.cid) + } + + if (obj.type != null && __BlockPresenceTypeValues[obj.type] !== 0) { + w.uint32(16) + BlockPresenceType.codec().encode(obj.type, w) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + cid: uint8ArrayAlloc(0), + type: BlockPresenceType.HaveBlock + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.cid = reader.bytes() + break + } + case 2: { + obj.type = BlockPresenceType.codec().decode(reader) + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, BlockPresence.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): BlockPresence => { + return decodeMessage(buf, BlockPresence.codec()) + } +} + +export interface BitswapMessage { + wantlist?: Wantlist + blocks: Block[] + blockPresences: BlockPresence[] + pendingBytes: number +} + +export namespace BitswapMessage { + let _codec: Codec + + export const codec = (): Codec => { + if (_codec == null) { + _codec = message((obj, w, opts = {}) => { + if (opts.lengthDelimited !== false) { + w.fork() + } + + if (obj.wantlist != null) { + w.uint32(10) + Wantlist.codec().encode(obj.wantlist, w) + } + + if (obj.blocks != null) { + for (const value of obj.blocks) { + w.uint32(26) + Block.codec().encode(value, w) + } + } + + if (obj.blockPresences != null) { + for (const value of obj.blockPresences) { + w.uint32(34) + BlockPresence.codec().encode(value, w) + } + } + + if ((obj.pendingBytes != null && obj.pendingBytes !== 0)) { + w.uint32(40) + w.int32(obj.pendingBytes) + } + + if (opts.lengthDelimited !== false) { + w.ldelim() + } + }, (reader, length) => { + const obj: any = { + blocks: [], + blockPresences: [], + pendingBytes: 0 + } + + const end = length == null ? reader.len : reader.pos + length + + while (reader.pos < end) { + const tag = reader.uint32() + + switch (tag >>> 3) { + case 1: { + obj.wantlist = Wantlist.codec().decode(reader, reader.uint32()) + break + } + case 3: { + obj.blocks.push(Block.codec().decode(reader, reader.uint32())) + break + } + case 4: { + obj.blockPresences.push(BlockPresence.codec().decode(reader, reader.uint32())) + break + } + case 5: { + obj.pendingBytes = reader.int32() + break + } + default: { + reader.skipType(tag & 7) + break + } + } + } + + return obj + }) + } + + return _codec + } + + export const encode = (obj: Partial): Uint8Array => { + return encodeMessage(obj, BitswapMessage.codec()) + } + + export const decode = (buf: Uint8Array | Uint8ArrayList): BitswapMessage => { + return decodeMessage(buf, BitswapMessage.codec()) + } +} diff --git a/packages/bitswap/src/peer-want-lists/index.ts b/packages/bitswap/src/peer-want-lists/index.ts new file mode 100644 index 00000000..bcd3b8fc --- /dev/null +++ b/packages/bitswap/src/peer-want-lists/index.ts @@ -0,0 +1,165 @@ +import { trackedPeerMap } from '@libp2p/peer-collections' +import { CID } from 'multiformats/cid' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { WantType } from '../pb/message.js' +import { Ledger } from './ledger.js' +import type { BitswapNotifyProgressEvents, WantListEntry } from '../index.js' +import type { Network } from '../network.js' +import type { BitswapMessage } from '../pb/message.js' +import type { ComponentLogger, Logger, Metrics, PeerId } from '@libp2p/interface' +import type { PeerMap } from '@libp2p/peer-collections' +import type { Blockstore } from 'interface-blockstore' +import type { AbortOptions } from 'it-length-prefixed-stream' +import type { ProgressOptions } from 'progress-events' + +export interface PeerWantListsInit { + maxSizeReplaceHasWithBlock?: number +} + +export interface PeerWantListsComponents { + blockstore: Blockstore + network: Network + metrics?: Metrics + logger: ComponentLogger +} + +export interface PeerLedger { + peer: PeerId + value: number + sent: number + received: number + exchanged: number +} + +export class PeerWantLists { + public blockstore: Blockstore + public network: Network + public readonly ledgerMap: PeerMap + private readonly maxSizeReplaceHasWithBlock?: number + private readonly log: Logger + + constructor (components: PeerWantListsComponents, init: PeerWantListsInit = {}) { + this.blockstore = components.blockstore + this.network = components.network + this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock + this.log = components.logger.forComponent('helia:bitswap:peer-want-lists') + + this.ledgerMap = trackedPeerMap({ + name: 'ipfs_bitswap_ledger_map', + metrics: components.metrics + }) + + this.network.addEventListener('bitswap:message', (evt) => { + this.receiveMessage(evt.detail.peer, evt.detail.message) + .catch(err => { + this.log.error('error receiving bitswap message from %p', evt.detail.peer, err) + }) + }) + this.network.addEventListener('peer:disconnected', evt => { + this.peerDisconnected(evt.detail) + }) + } + + ledgerForPeer (peerId: PeerId): PeerLedger | undefined { + const ledger = this.ledgerMap.get(peerId) + + if (ledger == null) { + return undefined + } + + return { + peer: ledger.peerId, + value: ledger.debtRatio(), + sent: ledger.bytesSent, + received: ledger.bytesReceived, + exchanged: ledger.exchangeCount + } + } + + wantListForPeer (peerId: PeerId): WantListEntry[] | undefined { + const ledger = this.ledgerMap.get(peerId) + + if (ledger == null) { + return undefined + } + + return [...ledger.wants.values()] + } + + peers (): PeerId[] { + return Array.from(this.ledgerMap.values()).map((l) => l.peerId) + } + + /** + * Handle incoming messages + */ + async receiveMessage (peerId: PeerId, message: BitswapMessage): Promise { + let ledger = this.ledgerMap.get(peerId) + + if (ledger == null) { + ledger = new Ledger({ + peerId, + blockstore: this.blockstore, + network: this.network + }, { + maxSizeReplaceHasWithBlock: this.maxSizeReplaceHasWithBlock + }) + this.ledgerMap.set(peerId, ledger) + } + + // record the amount of block data received + ledger.receivedBytes(message.blocks?.reduce((acc, curr) => acc + curr.data.byteLength, 0) ?? 0) + + if (message.wantlist != null) { + // if the message has a full wantlist, clear the current wantlist + if (message.wantlist.full === true) { + ledger.wants.clear() + } + + // clear cancelled wants and add new wants to the ledger + for (const entry of message.wantlist.entries) { + const cid = CID.decode(entry.cid) + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + + if (entry.cancel === true) { + this.log('peer %p cancelled want of block for %c', peerId, cid) + ledger.wants.delete(cidStr) + } else { + if (entry.wantType === WantType.WantHave) { + this.log('peer %p wanted block presence for %c', peerId, cid) + } else { + this.log('peer %p wanted block for %c', peerId, cid) + } + + ledger.wants.set(cidStr, { + cid, + priority: entry.priority, + wantType: entry.wantType ?? WantType.WantBlock, + sendDontHave: entry.sendDontHave ?? false + }) + } + } + } + + await ledger.sendBlocksToPeer() + } + + async receivedBlock (cid: CID, options: ProgressOptions & AbortOptions): Promise { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + const ledgers: Ledger[] = [] + + for (const ledger of this.ledgerMap.values()) { + if (ledger.wants.has(cidStr)) { + ledgers.push(ledger) + } + } + + await Promise.all( + ledgers.map(async (ledger) => ledger.sendBlocksToPeer(options)) + ) + } + + peerDisconnected (peerId: PeerId): void { + this.ledgerMap.delete(peerId) + } +} diff --git a/packages/bitswap/src/peer-want-lists/ledger.ts b/packages/bitswap/src/peer-want-lists/ledger.ts new file mode 100644 index 00000000..11d849ca --- /dev/null +++ b/packages/bitswap/src/peer-want-lists/ledger.ts @@ -0,0 +1,161 @@ +/* eslint-disable max-depth */ +import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK } from '../constants.js' +import { BlockPresenceType, type BitswapMessage, WantType } from '../pb/message.js' +import { cidToPrefix } from '../utils/cid-prefix.js' +import type { Network } from '../network.js' +import type { PeerId } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' +import type { AbortOptions } from 'it-length-prefixed-stream' +import type { CID } from 'multiformats/cid' + +export interface LedgerComponents { + peerId: PeerId + blockstore: Blockstore + network: Network +} + +export interface LedgerInit { + maxSizeReplaceHasWithBlock?: number +} + +export interface PeerWantListEntry { + /** + * The CID the peer has requested + */ + cid: CID + + /** + * The priority with which the remote should return the block + */ + priority: number + + /** + * If we want the block or if we want the remote to tell us if they have the + * block - note if the block is small they'll send it to us anyway. + */ + wantType: WantType + + /** + * Whether the remote should tell us if they have the block or not + */ + sendDontHave: boolean + + /** + * If we don't have the block and we've told them we don't have the block + */ + sentDontHave?: boolean +} + +export class Ledger { + public peerId: PeerId + private readonly blockstore: Blockstore + private readonly network: Network + public wants: Map + public exchangeCount: number + public bytesSent: number + public bytesReceived: number + public lastExchange?: number + private readonly maxSizeReplaceHasWithBlock: number + + constructor (components: LedgerComponents, init: LedgerInit) { + this.peerId = components.peerId + this.blockstore = components.blockstore + this.network = components.network + this.wants = new Map() + + this.exchangeCount = 0 + this.bytesSent = 0 + this.bytesReceived = 0 + this.maxSizeReplaceHasWithBlock = init.maxSizeReplaceHasWithBlock ?? DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK + } + + sentBytes (n: number): void { + this.exchangeCount++ + this.lastExchange = (new Date()).getTime() + this.bytesSent += n + } + + receivedBytes (n: number): void { + this.exchangeCount++ + this.lastExchange = (new Date()).getTime() + this.bytesReceived += n + } + + debtRatio (): number { + return (this.bytesSent / (this.bytesReceived + 1)) // +1 is to prevent division by zero + } + + public async sendBlocksToPeer (options?: AbortOptions): Promise { + const message: Pick = { + blockPresences: [], + blocks: [] + } + const sentBlocks = new Set() + + for (const [key, entry] of this.wants.entries()) { + const has = await this.blockstore.has(entry.cid, options) + + if (!has) { + // we don't have the requested block and the remote is not interested + // in us telling them that + if (!entry.sendDontHave) { + continue + } + + // we have already told them we don't have the block + if (entry.sentDontHave === true) { + continue + } + + entry.sentDontHave = true + message.blockPresences.push({ + cid: entry.cid.bytes, + type: BlockPresenceType.DontHaveBlock + }) + + continue + } + + const block = await this.blockstore.get(entry.cid, options) + + // do they want the block or just us to tell them we have the block + if (entry.wantType === WantType.WantHave) { + if (block.byteLength < this.maxSizeReplaceHasWithBlock) { + // if the block is small we just send it to them + sentBlocks.add(key) + message.blocks.push({ + data: block, + prefix: cidToPrefix(entry.cid) + }) + } else { + // otherwise tell them we have the block + message.blockPresences.push({ + cid: entry.cid.bytes, + type: BlockPresenceType.HaveBlock + }) + } + } else { + // they want the block, send it to them + sentBlocks.add(key) + message.blocks.push({ + data: block, + prefix: cidToPrefix(entry.cid) + }) + } + } + + // only send the message if we actually have something to send + if (message.blocks.length > 0 || message.blockPresences.length > 0) { + await this.network.sendMessage(this.peerId, message, options) + + // update accounting + this.sentBytes(message.blocks.reduce((acc, curr) => acc + curr.data.byteLength, 0)) + + // remove sent blocks from local copy of their want list - they can still + // re-request if required + for (const key of sentBlocks) { + this.wants.delete(key) + } + } + } +} diff --git a/packages/bitswap/src/session.ts b/packages/bitswap/src/session.ts new file mode 100644 index 00000000..d9f7c691 --- /dev/null +++ b/packages/bitswap/src/session.ts @@ -0,0 +1,150 @@ +import { CodeError } from '@libp2p/interface' +import { PeerSet } from '@libp2p/peer-collections' +import { PeerQueue } from '@libp2p/utils/peer-queue' +import map from 'it-map' +import merge from 'it-merge' +import pDefer, { type DeferredPromise } from 'p-defer' +import type { BitswapWantProgressEvents, BitswapSession as BitswapSessionInterface } from './index.js' +import type { Network } from './network.js' +import type { WantList } from './want-list.js' +import type { ComponentLogger, Logger, PeerId } from '@libp2p/interface' +import type { AbortOptions } from 'interface-store' +import type { CID } from 'multiformats/cid' +import type { ProgressOptions } from 'progress-events' + +export interface BitswapSessionComponents { + network: Network + wantList: WantList + logger: ComponentLogger +} + +export interface BitswapSessionInit extends AbortOptions { + root: CID + queryConcurrency: number + minProviders: number + maxProviders: number + connectedPeers: PeerId[] +} + +class BitswapSession implements BitswapSessionInterface { + public readonly root: CID + public readonly peers: PeerSet + private readonly log: Logger + private readonly wantList: WantList + private readonly network: Network + private readonly queue: PeerQueue + private readonly maxProviders: number + + constructor (components: BitswapSessionComponents, init: BitswapSessionInit) { + this.peers = new PeerSet() + this.root = init.root + this.maxProviders = init.maxProviders + this.log = components.logger.forComponent(`helia:bitswap:session:${init.root}`) + this.wantList = components.wantList + this.network = components.network + + this.queue = new PeerQueue({ + concurrency: init.queryConcurrency + }) + this.queue.addEventListener('error', (evt) => { + this.log.error('error querying peer for %c', this.root, evt.detail) + }) + } + + async want (cid: CID, options: AbortOptions & ProgressOptions = {}): Promise { + if (this.peers.size === 0) { + throw new CodeError('Bitswap session had no peers', 'ERR_NO_SESSION_PEERS') + } + + this.log('sending WANT-BLOCK for %c to', cid, this.peers) + + const result = await Promise.any( + [...this.peers].map(async peerId => { + return this.wantList.wantBlock(cid, { + peerId, + ...options + }) + }) + ) + + this.log('received block for %c from %p', cid, result.sender) + + // TODO findNewProviders when promise.any throws aggregate error and signal + // is not aborted + + return result.block + } + + async findNewProviders (cid: CID, count: number, options: AbortOptions = {}): Promise { + const deferred: DeferredPromise = pDefer() + let found = 0 + + this.log('find %d-%d new provider(s) for %c', count, this.maxProviders, cid) + + const source = merge( + [...this.wantList.peers.keys()], + map(this.network.findProviders(cid, options), prov => prov.id) + ) + + void Promise.resolve() + .then(async () => { + for await (const peerId of source) { + // eslint-disable-next-line no-loop-func + await this.queue.add(async () => { + try { + this.log('asking potential session peer %p if they have %c', peerId, cid) + const result = await this.wantList.wantPresence(cid, { + peerId, + ...options + }) + + if (!result.has) { + this.log('potential session peer %p did not have %c', peerId, cid) + return + } + + this.log('potential session peer %p had %c', peerId, cid) + found++ + + // add to list + this.peers.add(peerId) + + if (found === count) { + this.log('found %d session peers', found) + + deferred.resolve() + } + + if (found === this.maxProviders) { + this.log('found max provider session peers', found) + + this.queue.clear() + } + } catch (err: any) { + this.log.error('error querying potential session peer %p for %c', peerId, cid, err.errors ?? err) + } + }, { + peerId + }) + } + + this.log('found %d session peers total', found) + + if (count > 0) { + deferred.reject(new CodeError(`Found ${found} of ${count} providers`, 'ERR_NO_PROVIDERS_FOUND')) + } + }) + + return deferred.promise + } +} + +export async function createBitswapSession (components: BitswapSessionComponents, init: BitswapSessionInit): Promise { + const session = new BitswapSession(components, init) + + await session.findNewProviders(init.root, init.minProviders, { + signal: init.signal + }) + + return session +} diff --git a/packages/bitswap/src/stats.ts b/packages/bitswap/src/stats.ts new file mode 100644 index 00000000..9853dab0 --- /dev/null +++ b/packages/bitswap/src/stats.ts @@ -0,0 +1,67 @@ +import type { MetricGroup, Metrics, PeerId } from '@libp2p/interface' + +export interface StatsComponents { + metrics?: Metrics +} + +export class Stats { + private readonly blocksReceived?: MetricGroup + private readonly duplicateBlocksReceived?: MetricGroup + private readonly dataReceived?: MetricGroup + private readonly duplicateDataReceived?: MetricGroup + + constructor (components: StatsComponents) { + this.blocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_received_blocks') + this.duplicateBlocksReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_received_blocks') + this.dataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_data_received_bytes') + this.duplicateDataReceived = components.metrics?.registerMetricGroup('ipfs_bitswap_duplicate_data_received_bytes') + } + + updateBlocksReceived (count: number = 1, peerId?: PeerId): void { + const stats: Record = { + global: count + } + + if (peerId != null) { + stats[peerId.toString()] = count + } + + this.blocksReceived?.increment(stats) + } + + updateDuplicateBlocksReceived (count: number = 1, peerId?: PeerId): void { + const stats: Record = { + global: count + } + + if (peerId != null) { + stats[peerId.toString()] = count + } + + this.duplicateBlocksReceived?.increment(stats) + } + + updateDataReceived (bytes: number, peerId?: PeerId): void { + const stats: Record = { + global: bytes + } + + if (peerId != null) { + stats[peerId.toString()] = bytes + } + + this.dataReceived?.increment(stats) + } + + updateDuplicateDataReceived (bytes: number, peerId?: PeerId): void { + const stats: Record = { + global: bytes + } + + if (peerId != null) { + stats[peerId.toString()] = bytes + } + + this.duplicateDataReceived?.increment(stats) + } +} diff --git a/packages/bitswap/src/utils/cid-prefix.ts b/packages/bitswap/src/utils/cid-prefix.ts new file mode 100644 index 00000000..4e17fe93 --- /dev/null +++ b/packages/bitswap/src/utils/cid-prefix.ts @@ -0,0 +1,8 @@ +import ve from './varint-encoder.js' +import type { CID } from 'multiformats/cid' + +export function cidToPrefix (cid: CID): Uint8Array { + return ve([ + cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength + ]) +} diff --git a/packages/bitswap/src/utils/varint-decoder.ts b/packages/bitswap/src/utils/varint-decoder.ts new file mode 100644 index 00000000..0c8a0609 --- /dev/null +++ b/packages/bitswap/src/utils/varint-decoder.ts @@ -0,0 +1,19 @@ +import { decode, encodingLength } from 'uint8-varint' + +function varintDecoder (buf: Uint8Array): number[] { + if (!(buf instanceof Uint8Array)) { + throw new Error('arg needs to be a Uint8Array') + } + + const result: number[] = [] + + while (buf.length > 0) { + const num = decode(buf) + result.push(num) + buf = buf.slice(encodingLength(num)) + } + + return result +} + +export default varintDecoder diff --git a/packages/bitswap/src/utils/varint-encoder.ts b/packages/bitswap/src/utils/varint-encoder.ts new file mode 100644 index 00000000..2fd7b165 --- /dev/null +++ b/packages/bitswap/src/utils/varint-encoder.ts @@ -0,0 +1,18 @@ +import { encode, encodingLength } from 'uint8-varint' + +function varintEncoder (buf: number[]): Uint8Array { + let out = new Uint8Array(buf.reduce((acc, curr) => { + return acc + encodingLength(curr) + }, 0)) + let offset = 0 + + for (const num of buf) { + out = encode(num, out, offset) + + offset += encodingLength(num) + } + + return out +} + +export default varintEncoder diff --git a/packages/bitswap/src/want-list.ts b/packages/bitswap/src/want-list.ts new file mode 100644 index 00000000..5cd3d632 --- /dev/null +++ b/packages/bitswap/src/want-list.ts @@ -0,0 +1,529 @@ +import { AbortError } from '@libp2p/interface' +import { trackedPeerMap, PeerSet } from '@libp2p/peer-collections' +import { trackedMap } from '@libp2p/utils/tracked-map' +import all from 'it-all' +import filter from 'it-filter' +import map from 'it-map' +import { pipe } from 'it-pipe' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' +import pDefer from 'p-defer' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { DEFAULT_MESSAGE_SEND_DELAY } from './constants.js' +import { BlockPresenceType, WantType } from './pb/message.js' +import vd from './utils/varint-decoder.js' +import type { MultihashHasherLoader } from './index.js' +import type { BitswapNetworkWantProgressEvents, Network } from './network.js' +import type { BitswapMessage } from './pb/message.js' +import type { ComponentLogger, Metrics, PeerId, Startable, AbortOptions } from '@libp2p/interface' +import type { Logger } from '@libp2p/logger' +import type { PeerMap } from '@libp2p/peer-collections' +import type { DeferredPromise } from 'p-defer' +import type { ProgressOptions } from 'progress-events' + +export interface WantListComponents { + network: Network + logger: ComponentLogger + metrics?: Metrics +} + +export interface WantListInit { + sendMessagesDelay?: number + hashLoader?: MultihashHasherLoader +} + +export interface WantListEntry { + /** + * The CID we send to the remote + */ + cid: CID + + /** + * The priority with which the remote should return the block + */ + priority: number + + /** + * If we want the block or if we want the remote to tell us if they have the + * block - note if the block is small they'll send it to us anyway. + */ + wantType: WantType + + /** + * Whether we are cancelling the block want or not + */ + cancel: boolean + + /** + * Whether the remote should tell us if they have the block or not + */ + sendDontHave: boolean + + /** + * If this set has members, the want will only be sent to these peers + */ + session: PeerSet + + /** + * Promises returned from `.wantBlock` for this block + */ + blockWantListeners: Array> + + /** + * Promises returned from `.wantPresence` for this block + */ + blockPresenceListeners: Array> +} + +export interface WantOptions extends AbortOptions, ProgressOptions { + /** + * If set, this WantList entry will only be sent to this peer + */ + peerId?: PeerId + + /** + * Allow prioritising blocks + */ + priority?: number +} + +export interface WantBlockResult { + sender: PeerId + cid: CID + block: Uint8Array +} + +export interface WantDontHaveResult { + sender: PeerId + cid: CID + has: false +} + +export interface WantHaveResult { + sender: PeerId + cid: CID + has: true + block?: Uint8Array +} + +export type WantPresenceResult = WantDontHaveResult | WantHaveResult + +export class WantList implements Startable { + /** + * Tracks what CIDs we've previously sent to which peers + */ + public readonly peers: PeerMap> + public readonly wants: Map + private readonly network: Network + private readonly log: Logger + private readonly sendMessagesDelay: number + private sendMessagesTimeout?: ReturnType + private readonly hashLoader?: MultihashHasherLoader + + constructor (components: WantListComponents, init: WantListInit = {}) { + this.peers = trackedPeerMap({ + name: 'ipfs_bitswap_peers', + metrics: components.metrics + }) + this.wants = trackedMap({ + name: 'ipfs_bitswap_wantlist', + metrics: components.metrics + }) + this.network = components.network + this.sendMessagesDelay = init.sendMessagesDelay ?? DEFAULT_MESSAGE_SEND_DELAY + this.log = components.logger.forComponent('helia:bitswap:wantlist') + this.hashLoader = init.hashLoader + + this.network.addEventListener('bitswap:message', (evt) => { + this.receiveMessage(evt.detail.peer, evt.detail.message) + .catch(err => { + this.log.error('error receiving bitswap message from %p', evt.detail.peer, err) + }) + }) + this.network.addEventListener('peer:connected', evt => { + this.peerConnected(evt.detail) + .catch(err => { + this.log.error('error processing newly connected bitswap peer %p', evt.detail, err) + }) + }) + this.network.addEventListener('peer:disconnected', evt => { + this.peerDisconnected(evt.detail) + }) + } + + private async addEntry (cid: CID, options: WantOptions & { wantType: WantType.WantBlock }): Promise + private async addEntry (cid: CID, options: WantOptions & { wantType: WantType.WantHave }): Promise + private async addEntry (cid: CID, options: WantOptions & { wantType: WantType }): Promise { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + let entry = this.wants.get(cidStr) + + if (entry == null) { + entry = { + cid, + session: new PeerSet(), + priority: options.priority ?? 1, + wantType: options.wantType ?? WantType.WantBlock, + cancel: false, + sendDontHave: true, + blockWantListeners: [], + blockPresenceListeners: [] + } + + if (options.peerId != null) { + entry.session.add(options.peerId) + } + + this.wants.set(cidStr, entry) + } + + // upgrade want-have to want-block if the new want is a WantBlock but the + // previous want was a WantHave + if (entry.wantType === WantType.WantHave && options.wantType === WantType.WantBlock) { + entry.wantType = WantType.WantBlock + } + + // if this want was part of a session.. + if (entry.session.size > 0) { + // if the new want is also part of a session, expand the want session to + // include both sets of peers + if (options.peerId != null) { + entry.session.add(options.peerId) + } + + // if the new want is not part of a session, make this want a non-session + // want - nb. this will cause this WantList entry to be sent to every peer + // instead of just the ones in the session + if (options.peerId == null) { + entry.session.clear() + } + } + + // add a promise that will be resolved or rejected when the response arrives + let deferred: DeferredPromise + + if (options.wantType === WantType.WantBlock) { + const p = deferred = pDefer() + + entry.blockWantListeners.push(p) + } else { + const p = deferred = pDefer() + + entry.blockPresenceListeners.push(p) + } + + // reject the promise if the want is rejected + const abortListener = (): void => { + this.log('want for %c was aborted, cancelling want', cid) + + if (entry != null) { + entry.cancel = true + } + + deferred.reject(new AbortError('Want was aborted')) + } + options.signal?.addEventListener('abort', abortListener) + + // broadcast changes + clearTimeout(this.sendMessagesTimeout) + this.sendMessagesTimeout = setTimeout(() => { + void this.sendMessages() + .catch(err => { + this.log('error sending messages to peers', err) + }) + }, this.sendMessagesDelay) + + try { + return await deferred.promise + } finally { + // remove listener + options.signal?.removeEventListener('abort', abortListener) + // remove deferred promise + if (options.wantType === WantType.WantBlock) { + entry.blockWantListeners = entry.blockWantListeners.filter(recipient => recipient !== deferred) + } else { + entry.blockPresenceListeners = entry.blockPresenceListeners.filter(recipient => recipient !== deferred) + } + } + } + + private async sendMessages (): Promise { + for (const [peerId, sentWants] of this.peers) { + const sent = new Set() + const message: Partial = { + wantlist: { + full: false, + entries: pipe( + this.wants.entries(), + (source) => filter(source, ([key, entry]) => { + // skip session-only wants + if (entry.session.size > 0 && !entry.session.has(peerId)) { + return false + } + + const sentPreviously = sentWants.has(key) + + // don't cancel if we've not sent it to them before + if (entry.cancel) { + return sentPreviously + } + + // only send if we've not sent it to them before + return !sentPreviously + }), + (source) => map(source, ([key, entry]) => { + sent.add(key) + + return { + cid: entry.cid.bytes, + priority: entry.priority, + wantType: entry.wantType, + cancel: entry.cancel, + sendDontHave: entry.sendDontHave + } + }), + (source) => all(source) + ) + } + } + + if (message.wantlist?.entries.length === 0) { + return + } + + // add message to send queue + try { + await this.network.sendMessage(peerId, message) + + // update list of messages sent to remote + for (const key of sent) { + sentWants.add(key) + } + } catch (err: any) { + this.log.error('error sending full wantlist to new peer', err) + } + } + + // queued all message sends, remove cancelled wants from wantlist and sent + // wants + for (const [key, entry] of this.wants) { + if (entry.cancel) { + this.wants.delete(key) + + for (const sentWants of this.peers.values()) { + sentWants.delete(key) + } + } + } + } + + has (cid: CID): boolean { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + return this.wants.has(cidStr) + } + + /** + * Add a CID to the wantlist + */ + async wantPresence (cid: CID, options: WantOptions = {}): Promise { + if (options.peerId != null && this.peers.get(options.peerId) == null) { + const cidStr = uint8ArrayToString(cid.multihash.bytes, 'base64') + + try { + // if we don't have them as a peer, add them + this.peers.set(options.peerId, new Set([cidStr])) + + // sending WantHave directly to peer + await this.network.sendMessage(options.peerId, { + wantlist: { + full: false, + entries: [{ + cid: cid.bytes, + sendDontHave: true, + wantType: WantType.WantHave, + priority: 1 + }] + } + }) + } catch (err) { + // sending failed, remove them as a peer + this.peers.delete(options.peerId) + + throw err + } + } + + return this.addEntry(cid, { + ...options, + wantType: WantType.WantHave + }) + } + + /** + * Add a CID to the wantlist + */ + async wantBlock (cid: CID, options: WantOptions = {}): Promise { + return this.addEntry(cid, { + ...options, + wantType: WantType.WantBlock + }) + } + + /** + * Invoked when a message is received from a bitswap peer + */ + private async receiveMessage (sender: PeerId, message: BitswapMessage): Promise { + this.log('received message from %p', sender) + + // blocks received + const blockResults: WantBlockResult[] = [] + const presenceResults: WantPresenceResult[] = [] + + // process blocks + for (const block of message.blocks) { + if (block.prefix == null || block.data == null) { + continue + } + + this.log('received block') + const values = vd(block.prefix) + const cidVersion = values[0] + const multicodec = values[1] + const hashAlg = values[2] + // const hashLen = values[3] // We haven't need to use this so far + + const hasher = hashAlg === sha256.code ? sha256 : await this.hashLoader?.getHasher(hashAlg) + + if (hasher == null) { + this.log.error('unknown hash algorithm', hashAlg) + continue + } + + const hash = await hasher.digest(block.data) + const cid = CID.create(cidVersion === 0 ? 0 : 1, multicodec, hash) + + this.log('received block from %p for %c', sender, cid) + + blockResults.push({ + sender, + cid, + block: block.data + }) + + presenceResults.push({ + sender, + cid, + has: true + }) + } + + // process block presences + for (const { cid: cidBytes, type } of message.blockPresences) { + const cid = CID.decode(cidBytes) + + this.log('received %s from %p for %c', type, sender, cid) + + presenceResults.push({ + sender, + cid, + has: type === BlockPresenceType.HaveBlock + }) + } + + for (const result of blockResults) { + const cidStr = uint8ArrayToString(result.cid.multihash.bytes, 'base64') + const entry = this.wants.get(cidStr) + + if (entry == null) { + return + } + + const recipients = entry.blockWantListeners + entry.blockWantListeners = [] + recipients.forEach((p) => { + p.resolve(result) + }) + + // since we received the block, flip the cancel flag to send cancels to + // any peers on the next message sending iteration, this will remove it + // from the internal want list + entry.cancel = true + } + + for (const result of presenceResults) { + const cidStr = uint8ArrayToString(result.cid.multihash.bytes, 'base64') + const entry = this.wants.get(cidStr) + + if (entry == null) { + return + } + + const recipients = entry.blockPresenceListeners + entry.blockPresenceListeners = [] + recipients.forEach((p) => { + p.resolve(result) + }) + } + } + + /** + * Invoked when the network topology notices a new peer that supports Bitswap + */ + async peerConnected (peerId: PeerId): Promise { + const sentWants = new Set() + + // new peer, give them the full wantlist + const message: Partial = { + wantlist: { + full: true, + entries: pipe( + this.wants.entries(), + (source) => filter(source, ([key, entry]) => !entry.cancel && (entry.session.size > 0 && !entry.session.has(peerId))), + (source) => filter(source, ([key, entry]) => !entry.cancel), + (source) => map(source, ([key, entry]) => { + sentWants.add(key) + + return { + cid: entry.cid.bytes, + priority: 1, + wantType: WantType.WantBlock, + cancel: false, + sendDontHave: false + } + }), + (source) => all(source) + ) + } + } + + // only send the wantlist if we have something to send + if (message.wantlist?.entries.length === 0) { + this.peers.set(peerId, sentWants) + + return + } + + try { + await this.network.sendMessage(peerId, message) + + this.peers.set(peerId, sentWants) + } catch (err) { + this.log.error('error sending full wantlist to new peer %p', peerId, err) + } + } + + /** + * Invoked when the network topology notices peer that supports Bitswap has + * disconnected + */ + peerDisconnected (peerId: PeerId): void { + this.peers.delete(peerId) + } + + start (): void { + + } + + stop (): void { + this.peers.clear() + } +} diff --git a/packages/bitswap/test/bitswap.spec.ts b/packages/bitswap/test/bitswap.spec.ts new file mode 100644 index 00000000..f8a4a630 --- /dev/null +++ b/packages/bitswap/test/bitswap.spec.ts @@ -0,0 +1,437 @@ +import { DEFAULT_SESSION_MAX_PROVIDERS, DEFAULT_SESSION_MIN_PROVIDERS } from '@helia/interface' +import { start, stop } from '@libp2p/interface' +import { matchPeerId } from '@libp2p/interface-compliance-tests/matchers' +import { mockStream } from '@libp2p/interface-compliance-tests/mocks' +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { multiaddr } from '@multiformats/multiaddr' +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import delay from 'delay' +import { duplexPair } from 'it-pair/duplex' +import { pbStream } from 'it-protobuf-stream' +import { CID } from 'multiformats/cid' +import { sha256 } from 'multiformats/hashes/sha2' +import pWaitFor from 'p-wait-for' +import Sinon from 'sinon' +import { stubInterface } from 'sinon-ts' +import { Bitswap } from '../src/bitswap.js' +import { BitswapMessage, BlockPresenceType } from '../src/pb/message.js' +import { cidToPrefix } from '../src/utils/cid-prefix.js' +import type { BitswapMessageEventDetail } from '../src/network.js' +import type { Routing } from '@helia/interface/routing' +import type { Connection, Libp2p, PeerId } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' +import type { StubbedInstance } from 'sinon-ts' + +interface StubbedBitswapComponents { + peerId: PeerId + routing: StubbedInstance + blockstore: Blockstore + libp2p: StubbedInstance +} + +describe('bitswap', () => { + let components: StubbedBitswapComponents + let bitswap: Bitswap + let cid: CID + let block: Uint8Array + + beforeEach(async () => { + block = Uint8Array.from([0, 1, 2, 3, 4]) + const mh = await sha256.digest(block) + cid = CID.createV0(mh).toV1() + + components = { + peerId: await createEd25519PeerId(), + routing: stubInterface(), + blockstore: new MemoryBlockstore(), + libp2p: stubInterface() + } + + bitswap = new Bitswap({ + ...components, + logger: defaultLogger() + }) + + components.libp2p.getConnections.returns([]) + + await start(bitswap) + }) + + afterEach(async () => { + if (bitswap != null) { + await stop(bitswap) + } + }) + + describe('session', () => { + it('should create a session', async () => { + const connectedPeer = await createEd25519PeerId() + + // notify topology of connected peer that supports bitswap + components.libp2p.register.getCall(0).args[1]?.onConnect?.(connectedPeer, stubInterface({ + remotePeer: connectedPeer + })) + + // the current peer does not have the block + stubPeerResponse(components.libp2p, connectedPeer, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + // providers found via routing + const providers = await Promise.all( + new Array(10).fill(0).map(async (_, i) => { + return { + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr(`/ip4/4${i}.4${i}.4${i}.4${i}/tcp/${1234 + i}`) + ], + protocols: ['transport-bitswap'] + } + }) + ) + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + // stub first three provider responses, all but #3 have the block, second + // provider sends the block in the response + stubPeerResponse(components.libp2p, providers[0].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.HaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[1].id, { + blockPresences: [], + blocks: [{ + prefix: cidToPrefix(cid), + data: block + }], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[2].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[3].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.HaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[4].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.HaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + stubPeerResponse(components.libp2p, providers[5].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.HaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + const session = await bitswap.createSession(cid) + expect(session.peers.size).to.equal(DEFAULT_SESSION_MIN_PROVIDERS) + expect([...session.peers].map(p => p.toString())).to.include(providers[0].id.toString()) + + // dialed connected peer first + expect(connectedPeer.equals(components.libp2p.dialProtocol.getCall(0).args[0].toString())).to.be.true() + + // dialed first provider second + expect(providers[0].id.equals(components.libp2p.dialProtocol.getCall(1).args[0].toString())).to.be.true() + + // the query continues after the session is ready + await pWaitFor(() => { + return session.peers.size === DEFAULT_SESSION_MAX_PROVIDERS + }) + + // should have continued querying until we reach DEFAULT_SESSION_MAX_PROVIDERS + expect(providers[1].id.equals(components.libp2p.dialProtocol.getCall(2).args[0].toString())).to.be.true() + expect(providers[2].id.equals(components.libp2p.dialProtocol.getCall(3).args[0].toString())).to.be.true() + + // should have stopped at DEFAULT_SESSION_MAX_PROVIDERS + expect(session.peers.size).to.equal(DEFAULT_SESSION_MAX_PROVIDERS) + }) + + it('should error when creating a session when no peers or providers have the block', async () => { + const connectedPeer = await createEd25519PeerId() + + // notify topology of connected peer that supports bitswap + components.libp2p.register.getCall(0).args[1]?.onConnect?.(connectedPeer, stubInterface({ + remotePeer: connectedPeer + })) + + // the current peer does not have the block + stubPeerResponse(components.libp2p, connectedPeer, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + // providers found via routing + const providers = [{ + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/41.41.41.41/tcp/1234') + ], + protocols: ['transport-bitswap'] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + // the provider doesn't have the block + stubPeerResponse(components.libp2p, providers[0].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + await expect(bitswap.createSession(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_PROVIDERS_FOUND') + }) + + it('should error when creating a session when no providers have the block', async () => { + // providers found via routing + const providers = [{ + id: await createEd25519PeerId(), + multiaddrs: [ + multiaddr('/ip4/41.41.41.41/tcp/1234') + ], + protocols: ['transport-bitswap'] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + // the provider doesn't have the block + stubPeerResponse(components.libp2p, providers[0].id, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + await expect(bitswap.createSession(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_PROVIDERS_FOUND') + }) + + it('should error when creating a session when no peers have the block', async () => { + const connectedPeer = await createEd25519PeerId() + + // notify topology of connected peer that supports bitswap + components.libp2p.register.getCall(0).args[1]?.onConnect?.(connectedPeer, stubInterface({ + remotePeer: connectedPeer + })) + + // the current peer does not have the block + stubPeerResponse(components.libp2p, connectedPeer, { + blockPresences: [{ + cid: cid.bytes, + type: BlockPresenceType.DontHaveBlock + }], + blocks: [], + pendingBytes: 0 + }) + + components.routing.findProviders.withArgs(cid).returns((async function * () {})()) + + await expect(bitswap.createSession(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_PROVIDERS_FOUND') + }) + + it('should error when creating a session when there are peers and no providers found', async () => { + components.routing.findProviders.withArgs(cid).returns((async function * () {})()) + + await expect(bitswap.createSession(cid)).to.eventually.be.rejected + .with.property('code', 'ERR_NO_PROVIDERS_FOUND') + }) + }) + + describe('want', () => { + it('should want a block that is available on the network', async () => { + const remotePeer = await createEd25519PeerId() + const findProvsSpy = Sinon.spy(bitswap.network, 'findAndConnect') + + const p = bitswap.want(cid) + + // provider sends message + bitswap.network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [{ + prefix: cidToPrefix(cid), + data: block + }], + blockPresences: [], + pendingBytes: 0 + } + } + }) + + const b = await p + + // should have added cid to wantlist and searched for providers + expect(findProvsSpy.called).to.be.true() + + // should have cancelled the notification request + expect(b).to.equalBytes(block) + }) + + it('should abort wanting a block that is not available on the network', async () => { + const p = bitswap.want(cid, { + signal: AbortSignal.timeout(100) + }) + + await expect(p).to.eventually.be.rejected + .with.property('code', 'ABORT_ERR') + }) + + it('should notify peers we have a block', async () => { + const receivedBlockSpy = Sinon.spy(bitswap.peerWantLists, 'receivedBlock') + + await bitswap.notify(cid, block) + + expect(receivedBlockSpy.called).to.be.true() + }) + }) + + describe('wantlist', () => { + it('should remove CIDs from the wantlist when the block arrives', async () => { + const remotePeer = await createEd25519PeerId() + expect(bitswap.getWantlist()).to.be.empty() + + const p = bitswap.want(cid) + + expect(bitswap.getWantlist().map(w => w.cid)).to.include(cid) + + // provider sends message + bitswap.network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [{ + prefix: cidToPrefix(cid), + data: block + }], + blockPresences: [], + pendingBytes: 0 + } + } + }) + + const b = await p + + expect(bitswap.getWantlist()).to.be.empty() + expect(b).to.equalBytes(block) + }) + + it('should remove CIDs from the wantlist when the want is aborted', async () => { + expect(bitswap.getWantlist()).to.be.empty() + + const p = bitswap.want(cid, { + signal: AbortSignal.timeout(100) + }) + + expect(bitswap.getWantlist().map(w => w.cid)).to.include(cid) + + await expect(p).to.eventually.be.rejected + .with.property('code', 'ABORT_ERR') + + expect(bitswap.getWantlist()).to.be.empty() + }) + }) + + describe('peer wantlist', () => { + it('should return a peer wantlist', async () => { + const remotePeer = await createEd25519PeerId() + + // don't have this peer yet + expect(bitswap.getPeerWantlist(remotePeer)).to.be.undefined() + + // peers sends message with wantlist + bitswap.network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + wantlist: { + full: false, + entries: [{ + cid: cid.bytes, + priority: 100 + }] + }, + blockPresences: [], + blocks: [], + pendingBytes: 0 + } + } + }) + + expect(bitswap.getPeerWantlist(remotePeer)?.map(entry => entry.cid)).to.deep.equal([cid]) + }) + }) +}) + +function stubPeerResponse (libp2p: StubbedInstance, peerId: PeerId, response: BitswapMessage): void { + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + + libp2p.dialProtocol.withArgs(matchPeerId(peerId)).resolves(remoteStream) + + const connection = stubInterface({ + remotePeer: peerId + }) + + const pbstr = pbStream(localStream).pb(BitswapMessage) + void pbstr.read().then(async message => { + // simulate network latency + await delay(10) + + // after reading message from remote, open a new stream on the remote and + // send the response + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + + const onStream = libp2p.handle.getCall(0).args[1] + onStream({ stream: remoteStream, connection }) + + const pbstr = pbStream(localStream).pb(BitswapMessage) + await pbstr.write(response) + }) +} diff --git a/packages/bitswap/test/network.spec.ts b/packages/bitswap/test/network.spec.ts new file mode 100644 index 00000000..ad958e4a --- /dev/null +++ b/packages/bitswap/test/network.spec.ts @@ -0,0 +1,506 @@ +import { CustomEvent, isPeerId } from '@libp2p/interface' +import { mockStream } from '@libp2p/interface-compliance-tests/mocks' +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { multiaddr } from '@multiformats/multiaddr' +import { expect } from 'aegir/chai' +import delay from 'delay' +import all from 'it-all' +import { lpStream } from 'it-length-prefixed-stream' +import { duplexPair } from 'it-pair/duplex' +import { pbStream } from 'it-protobuf-stream' +import { CID } from 'multiformats/cid' +import { pEvent } from 'p-event' +import pRetry from 'p-retry' +import Sinon from 'sinon' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { BITSWAP_120 } from '../src/constants.js' +import { Network } from '../src/network.js' +import { BitswapMessage, BlockPresenceType } from '../src/pb/message.js' +import { cidToPrefix } from '../src/utils/cid-prefix.js' +import type { Routing } from '@helia/interface/routing' +import type { Connection, Libp2p, PeerId, IdentifyResult } from '@libp2p/interface' + +interface StubbedNetworkComponents { + routing: StubbedInstance + libp2p: StubbedInstance +} + +describe('network', () => { + let network: Network + let components: StubbedNetworkComponents + + beforeEach(async () => { + components = { + routing: stubInterface(), + libp2p: stubInterface({ + getConnections: () => [] + }) + } + + network = new Network({ + ...components, + logger: defaultLogger() + }, { + messageReceiveTimeout: 100 + }) + + await network.start() + }) + + afterEach(async () => { + if (network != null) { + await network.stop() + } + }) + + it('should not connect if not running', async () => { + await network.stop() + + const peerId = await createEd25519PeerId() + + await expect(network.connectTo(peerId)) + .to.eventually.be.rejected.with.property('code', 'ERR_NOT_STARTED') + }) + + it('should register protocol handlers', () => { + expect(components.libp2p.handle.called).to.be.true() + expect(components.libp2p.register.calledWith(BITSWAP_120)).to.be.true() + }) + + it('should deregister protocol handlers', async () => { + await network.stop() + + expect(components.libp2p.unhandle.called).to.be.true() + }) + + it('should start twice', async () => { + expect(components.libp2p.handle.calledOnce).to.be.true() + + await network.start() + + expect(components.libp2p.handle.calledOnce).to.be.true() + }) + + it('should emit a bitswap:message event when receiving an incoming message', async () => { + const remotePeer = await createEd25519PeerId() + const connection = stubInterface({ + remotePeer + }) + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + const handler = components.libp2p.handle.getCall(0).args[1] + + const messageEventPromise = pEvent<'bitswap:message', CustomEvent<{ peer: PeerId, message: BitswapMessage }>>(network, 'bitswap:message') + + handler({ + stream: remoteStream, + connection + }) + + const pbstr = pbStream(localStream).pb(BitswapMessage) + await pbstr.write({ + blockPresences: [], + blocks: [], + wantlist: { + full: true, + entries: [] + }, + pendingBytes: 0 + }) + + const event = await messageEventPromise + + expect(event.detail.peer.toString()).to.equal(remotePeer.toString()) + expect(event.detail).to.have.nested.property('message.wantlist.full', true) + }) + + it('should close the stream if parsing an incoming message fails', async () => { + const remotePeer = await createEd25519PeerId() + const connection = stubInterface({ + remotePeer + }) + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + const handler = components.libp2p.handle.getCall(0).args[1] + + const spy = Sinon.spy(remoteStream, 'abort') + + handler({ + stream: remoteStream, + connection + }) + + const lpstr = lpStream(localStream) + + // garbage data, cannot be unmarshalled as protobuf + await lpstr.write(Uint8Array.from([0, 1, 2, 3])) + + await pRetry(() => { + expect(spy.called).to.be.true() + }) + }) + + it('should close the stream if no message is received', async () => { + const remotePeer = await createEd25519PeerId() + const connection = stubInterface({ + remotePeer + }) + const [, remoteDuplex] = duplexPair() + const remoteStream = mockStream(remoteDuplex) + const handler = components.libp2p.handle.getCall(0).args[1] + + const spy = Sinon.spy(remoteStream, 'abort') + + handler({ + stream: remoteStream, + connection + }) + + await pRetry(() => { + expect(spy.called).to.be.true() + }) + }) + + it('should find providers', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + const output = await all(network.findProviders(cid)) + + expect(output).to.have.lengthOf(1) + expect(output[0].id.toString()).to.equal(peerId.toString()) + expect(output[0].multiaddrs).to.have.lengthOf(1) + expect(output[0].multiaddrs[0].toString()).to.equal('/ip4/127.0.0.1/tcp/4001') + }) + + it('should ignore providers with only transient addresses', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001/p2p-circuit') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + const output = await all(network.findProviders(cid)) + + expect(output).to.be.empty() + }) + + it('should find providers with only transient addresses when running on transient connections', async () => { + await network.stop() + network = new Network({ + ...components, + logger: defaultLogger() + }, { + runOnTransientConnections: true + }) + + await network.start() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001/p2p-circuit') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + const output = await all(network.findProviders(cid)) + + expect(output).to.have.lengthOf(1) + expect(output[0].id.toString()).to.equal(peerId.toString()) + expect(output[0].multiaddrs).to.have.lengthOf(1) + expect(output[0].multiaddrs[0].toString()).to.equal('/ip4/127.0.0.1/tcp/4001/p2p-circuit') + }) + + it('should find and connect to a peer', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + components.libp2p.dial.callsFake(async (peerId) => { + // fake a network delay + await delay(100) + + const connection = stubInterface() + + // simulate identify having run + setTimeout(() => { + const call = components.libp2p.addEventListener.getCall(0) + + expect(call.args[0]).to.equal('peer:identify') + const callback = call.args[1] + + if (isPeerId(peerId) && typeof callback === 'function') { + callback(new CustomEvent('peer:identify', { + detail: { + peerId, + protocols: [ + BITSWAP_120 + ], + listenAddrs: [], + connection + } + })) + } + }, 100) + + return connection + }) + + await network.findAndConnect(cid) + + expect(components.libp2p.dial.calledWith(peerId)).to.be.true() + }) + + it('should find and connect to a peer with only a transient address when running on transient connections', async () => { + await network.stop() + network = new Network({ + ...components, + logger: defaultLogger() + }, { + runOnTransientConnections: true + }) + await network.start() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + const providers = [{ + id: peerId, + multiaddrs: [ + multiaddr('/ip4/127.0.0.1/tcp/4001/p2p-circuit') + ] + }] + + components.routing.findProviders.withArgs(cid).returns((async function * () { + yield * providers + })()) + + components.libp2p.dial.callsFake(async (peerId) => { + // fake a network delay + await delay(100) + + const connection = stubInterface() + + // simulate identify having run + setTimeout(() => { + const call = components.libp2p.addEventListener.getCall(0) + + expect(call.args[0]).to.equal('peer:identify') + const callback = call.args[1] + + if (isPeerId(peerId) && typeof callback === 'function') { + callback(new CustomEvent('peer:identify', { + detail: { + peerId, + protocols: [ + BITSWAP_120 + ], + listenAddrs: [], + connection + } + })) + } + }, 100) + + return connection + }) + + await network.findAndConnect(cid) + + expect(components.libp2p.dial.calledWith(peerId)).to.be.true() + }) + + it('should send a message', async () => { + const peerId = await createEd25519PeerId() + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + + components.libp2p.dialProtocol.withArgs(peerId, BITSWAP_120).resolves(remoteStream) + + void network.sendMessage(peerId, { + blocks: [], + blockPresences: [], + wantlist: { + full: true, + entries: [] + }, + pendingBytes: 0 + }) + + const pbstr = pbStream(localStream).pb(BitswapMessage) + const message = await pbstr.read() + + expect(message).to.have.nested.property('wantlist.full').that.is.true() + }) + + it('should merge messages sent to the same peer', async () => { + await network.stop() + network = new Network({ + ...components, + logger: defaultLogger() + }, { + messageSendConcurrency: 1 + }) + await network.start() + + const peerId = await createEd25519PeerId() + const [localDuplex, remoteDuplex] = duplexPair() + const localStream = mockStream(localDuplex) + const remoteStream = mockStream(remoteDuplex) + + components.libp2p.dialProtocol.withArgs(peerId, BITSWAP_120).resolves(remoteStream) + + const cid1 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3A') + const cid2 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3B') + const cid3 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3C') + const cid4 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3D') + const cid5 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3E') + const cid6 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const cid7 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3G') + + const messageA = { + blocks: [{ + prefix: cidToPrefix(cid1), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }], + blockPresences: [{ + cid: cid3.bytes, + type: BlockPresenceType.DontHaveBlock + }, { + cid: cid5.bytes, + type: BlockPresenceType.DontHaveBlock + }], + wantlist: { + full: true, + entries: [{ + cid: cid5.bytes, + priority: 5 + }, { + cid: cid6.bytes, + priority: 100 + }] + }, + pendingBytes: 5 + } + + const messageB = { + blocks: [{ + prefix: cidToPrefix(cid2), + data: Uint8Array.from([5, 6, 7, 8]) + }], + blockPresences: [{ + cid: cid4.bytes, + type: BlockPresenceType.DontHaveBlock + }, { + cid: cid5.bytes, + type: BlockPresenceType.HaveBlock + }], + wantlist: { + full: false, + entries: [{ + cid: cid6.bytes, + priority: 0 + }, { + cid: cid7.bytes, + priority: 0 + }] + }, + pendingBytes: 7 + } + + // block the queue with a slow request + const slowPeer = await createEd25519PeerId() + components.libp2p.dialProtocol.withArgs(slowPeer).callsFake(async () => { + await delay(100) + throw new Error('Urk!') + }) + void network.sendMessage(slowPeer, { + blocks: [{ + prefix: cidToPrefix(cid1), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }] + }).catch(() => {}) + + // send two messages while the queue is blocked + void network.sendMessage(peerId, messageA) + void network.sendMessage(peerId, messageB) + + // wait for long enough that we are sure we don't dial peerId twice + await delay(500) + + // one dial for slowPeer, one for peerId + expect(components.libp2p.dialProtocol).to.have.property('callCount', 2, 'made too many dials') + + const pbstr = pbStream(localStream).pb(BitswapMessage) + const message = await pbstr.read() + + expect(message).to.have.deep.property('blocks', [ + ...messageA.blocks, + ...messageB.blocks + ]) + expect(message).to.have.deep.property('blockPresences', [{ + cid: cid3.bytes, + type: BlockPresenceType.DontHaveBlock + }, { + cid: cid5.bytes, + type: BlockPresenceType.HaveBlock + }, { + cid: cid4.bytes, + type: BlockPresenceType.DontHaveBlock + }]) + expect(message).to.have.deep.property('wantlist', { + full: true, + entries: [{ + cid: cid5.bytes, + priority: 5 + }, { + cid: cid6.bytes, + priority: 100 + }, { + cid: cid7.bytes, + priority: 0 + }] + }) + expect(message).to.have.property('pendingBytes', messageA.pendingBytes + messageB.pendingBytes) + }) +}) diff --git a/packages/bitswap/test/peer-want-list.spec.ts b/packages/bitswap/test/peer-want-list.spec.ts new file mode 100644 index 00000000..2ed3e0ed --- /dev/null +++ b/packages/bitswap/test/peer-want-list.spec.ts @@ -0,0 +1,600 @@ +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import delay from 'delay' +import { CID } from 'multiformats/cid' +import pRetry from 'p-retry' +import Sinon from 'sinon' +import { stubInterface } from 'sinon-ts' +import { DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK, DEFAULT_MESSAGE_SEND_DELAY } from '../src/constants.js' +import { Network } from '../src/network.js' +import { BlockPresenceType, WantType } from '../src/pb/message.js' +import { PeerWantLists } from '../src/peer-want-lists/index.js' +import ve from '../src/utils/varint-encoder.js' +import type { Routing } from '@helia/interface' +import type { Libp2p, ComponentLogger, PeerId } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' + +interface PeerWantListsComponentStubs { + peerId: PeerId + blockstore: Blockstore + network: Network + logger: ComponentLogger +} + +describe('peer-want-lists', () => { + let components: PeerWantListsComponentStubs + let wantLists: PeerWantLists + let network: Network + + beforeEach(async () => { + const logger = defaultLogger() + network = new Network({ + routing: stubInterface(), + logger, + libp2p: stubInterface({ + getConnections: () => [] + }) + }) + + components = { + peerId: await createEd25519PeerId(), + blockstore: new MemoryBlockstore(), + network, + logger: defaultLogger() + } + + wantLists = new PeerWantLists(components) + }) + + it('should keep a ledger for a peer', async () => { + const remotePeer = await createEd25519PeerId() + + expect(wantLists.ledgerForPeer(remotePeer)).to.be.undefined('should not have list initially') + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + full: true, + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } + } + }) + + const ledger = wantLists.ledgerForPeer(remotePeer) + + expect(ledger).to.have.property('peer', remotePeer) + expect(ledger).to.have.property('value', 0) + expect(ledger).to.have.property('sent', 0) + expect(ledger).to.have.property('received', 0) + expect(ledger).to.have.property('exchanged', 1) + }) + + it('should replace the wantlist for a peer when the full list is received', async () => { + const remotePeer = await createEd25519PeerId() + + const cid1 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const cid2 = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + + // first wantlist + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid1.bytes, + priority: 1 + }] + } + } + } + }) + + let entries = wantLists.wantListForPeer(remotePeer) + + expect(entries?.map(entry => entry.cid.toString())).to.include(cid1.toString()) + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + full: true, + entries: [{ + cid: cid2.bytes, + priority: 1 + }] + } + } + } + }) + + entries = wantLists.wantListForPeer(remotePeer) + + // should only have CIDs from the second message + expect(entries?.map(entry => entry.cid.toString())).to.not.include(cid1.toString()) + expect(entries?.map(entry => entry.cid.toString())).to.include(cid2.toString()) + }) + + it('should merge the wantlist for a peer when a partial list is received', async () => { + const remotePeer = await createEd25519PeerId() + + const cid1 = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const cid2 = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + + // first wantlist + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid1.bytes, + priority: 1 + }] + } + } + } + }) + + let entries = wantLists.wantListForPeer(remotePeer) + + expect(entries?.map(entry => entry.cid.toString())).to.include(cid1.toString()) + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid2.bytes, + priority: 1 + }] + } + } + } + }) + + entries = wantLists.wantListForPeer(remotePeer) + + // should have both CIDs + expect(entries?.map(entry => entry.cid.toString())).to.include(cid1.toString()) + expect(entries?.map(entry => entry.cid.toString())).to.include(cid2.toString()) + }) + + it('should record the amount of incoming data', async () => { + const remotePeer = await createEd25519PeerId() + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [{ + prefix: Uint8Array.from([0, 1, 2, 3, 4]), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + prefix: Uint8Array.from([0, 1, 2]), + data: Uint8Array.from([0, 1, 2]) + }], + blockPresences: [], + pendingBytes: 0 + } + } + }) + + const ledger = wantLists.ledgerForPeer(remotePeer) + + expect(ledger).to.have.property('received', 8) + }) + + it('should send requested blocks to peer', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3, 4]) + + // we have block + await components.blockstore.put(cid, block) + + // incoming message + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } + } + }) + + // wait for network send + await pRetry(() => { + if (!sendMessageStub.called) { + throw new Error('Network message not sent') + } + }) + + const message = sendMessageStub.getCall(0).args[1] + + expect(message.blocks).to.have.lengthOf(1) + expect(message.blocks?.[0].data).to.equalBytes(block) + expect(message.blocks?.[0].prefix).to.equalBytes(ve([ + cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength + ])) + + // have to wait for network send + await delay(DEFAULT_MESSAGE_SEND_DELAY * 3) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.not.include(cid.toString()) + }) + + it('should send requested block presences to peer', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from(new Array(DEFAULT_MAX_SIZE_REPLACE_HAS_WITH_BLOCK + 1)) + + // we have block + await components.blockstore.put(cid, block) + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave + }] + } + } + } + }) + + // wait for network send + await pRetry(() => { + if (!sendMessageStub.called) { + throw new Error('Network message not sent') + } + }) + + const message = sendMessageStub.getCall(0).args[1] + + expect(message.blocks).to.be.empty('should not have sent blocks') + expect(message.blockPresences).to.have.lengthOf(1) + expect(message.blockPresences?.[0].cid).to.equalBytes(cid.bytes) + expect(message.blockPresences?.[0].type).to.equal(BlockPresenceType.HaveBlock, 'should have sent HaveBlock presence') + }) + + it('should send requested lack of block presences to peer', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() + const remotePeer = await createEd25519PeerId() + + // CID for a block we don't have + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantBlock, + sendDontHave: true + }] + } + } + } + }) + + // wait for network send + await pRetry(() => { + if (!sendMessageStub.called) { + throw new Error('Network message not sent') + } + }) + + const message = sendMessageStub.getCall(0).args[1] + + expect(message.blocks).to.be.empty('should not have sent blocks') + expect(message.blockPresences).to.have.lengthOf(1) + expect(message.blockPresences?.[0].cid).to.equalBytes(cid.bytes) + expect(message.blockPresences?.[0].type).to.equal(BlockPresenceType.DontHaveBlock, 'should have sent DontHaveBlock presence') + }) + + it('should send requested blocks to peer when presence was requested but block size is less than maxSizeReplaceHasWithBlock', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3, 4]) + + // we have block + await components.blockstore.put(cid, block) + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave + }] + } + } + } + }) + + // wait for network send + await pRetry(() => { + if (!sendMessageStub.called) { + throw new Error('Network message not sent') + } + }) + + const message = sendMessageStub.getCall(0).args[1] + + expect(message.blockPresences).to.be.empty() + expect(message.blocks).to.have.lengthOf(1) + expect(message.blocks?.[0].data).to.equalBytes(block) + expect(message.blocks?.[0].prefix).to.equalBytes(ve([ + cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength + ])) + + // have to wait for network send + await delay(1) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.not.include(cid.toString()) + }) + + it('should send requested block presences to peer for blocks we don\'t have', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + wantType: WantType.WantHave, + sendDontHave: true + }] + } + } + } + }) + + // wait for network send + await pRetry(() => { + if (!sendMessageStub.called) { + throw new Error('Network message not sent') + } + }) + + const message = sendMessageStub.getCall(0).args[1] + + expect(message.blocks).to.be.empty('should not have sent blocks') + expect(message.blockPresences).to.have.lengthOf(1) + expect(message.blockPresences?.[0].cid).to.equalBytes(cid.bytes) + expect(message.blockPresences?.[0].type).to.equal(BlockPresenceType.DontHaveBlock, 'should have sent DontHaveBlock presence') + }) + + it('should remove wants when peer cancels', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } + } + }) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.include(cid.toString()) + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1, + cancel: true + }] + } + } + } + }) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())).to.not.include(cid.toString()) + }) + + it('should remove wantlist and ledger when peer disconnects', async () => { + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3, 4]) + + // we have block + await components.blockstore.put(cid, block) + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } + } + }) + + expect(wantLists.ledgerForPeer(remotePeer)).to.be.ok() + expect(wantLists.wantListForPeer(remotePeer)).to.be.ok() + + wantLists.peerDisconnected(remotePeer) + + expect(wantLists.ledgerForPeer(remotePeer)).to.be.undefined() + expect(wantLists.wantListForPeer(remotePeer)).to.be.undefined() + }) + + it('should return peers with want lists', async () => { + const remotePeer = await createEd25519PeerId() + + expect(wantLists.peers()).to.be.empty() + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [{ + prefix: Uint8Array.from([0, 1, 2, 3, 4]), + data: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + prefix: Uint8Array.from([0, 1, 2]), + data: Uint8Array.from([0, 1, 2]) + }], + blockPresences: [], + pendingBytes: 0 + } + } + }) + + expect(wantLists.peers().map(p => p.toString())).to.include(remotePeer.toString()) + }) + + it('should send requested blocks to peer when they are received', async () => { + const sendMessageStub = network.sendMessage = Sinon.stub() + const remotePeer = await createEd25519PeerId() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3, 4]) + + network.safeDispatchEvent('bitswap:message', { + detail: { + peer: remotePeer, + message: { + blocks: [], + blockPresences: [], + pendingBytes: 0, + wantlist: { + entries: [{ + cid: cid.bytes, + priority: 1 + }] + } + } + } + }) + + expect(wantLists.wantListForPeer(remotePeer)?.map(e => e.cid.toString())).to.include(cid.toString()) + + // now we have block + await components.blockstore.put(cid, block) + + // we received it + await wantLists.receivedBlock(cid, {}) + + // wait for network send + await pRetry(() => { + if (!sendMessageStub.called) { + throw new Error('Network message not sent') + } + }) + + const message = sendMessageStub.getCall(0).args[1] + + expect(message.blocks).to.have.lengthOf(1) + expect(message.blocks?.[0].data).to.equalBytes(block) + expect(message.blocks?.[0].prefix).to.equalBytes(ve([ + cid.version, cid.code, cid.multihash.code, cid.multihash.digest.byteLength + ])) + + // have to wait for network send + await delay(1) + + expect(wantLists.wantListForPeer(remotePeer)?.map(entry => entry.cid.toString())) + .to.not.include(cid.toString()) + + // should only have sent one message + await delay(100) + expect(sendMessageStub.callCount).to.equal(1) + }) +}) diff --git a/packages/bitswap/test/session.spec.ts b/packages/bitswap/test/session.spec.ts new file mode 100644 index 00000000..f91c07b2 --- /dev/null +++ b/packages/bitswap/test/session.spec.ts @@ -0,0 +1,75 @@ +import { defaultLogger } from '@libp2p/logger' +import { PeerMap } from '@libp2p/peer-collections' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { createBitswapSession } from '../src/session.js' +import type { BitswapSession } from '../src/index.js' +import type { Network } from '../src/network.js' +import type { WantList } from '../src/want-list.js' + +interface StubbedBitswapSessionComponents { + network: StubbedInstance + wantList: StubbedInstance +} + +describe('session', () => { + let components: StubbedBitswapSessionComponents + let session: BitswapSession + let cid: CID + + beforeEach(() => { + cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + + components = { + network: stubInterface(), + wantList: stubInterface({ + peers: new PeerMap() + }) + } + }) + + it('should only query session peers', async () => { + const peerId = await createEd25519PeerId() + const data = new Uint8Array([0, 1, 2, 3, 4]) + + components.network.findProviders.returns(async function * () { + yield { + id: peerId, + multiaddrs: [], + protocols: [''] + } + }()) + + components.wantList.wantPresence.resolves({ + sender: peerId, + cid, + has: true + }) + + components.wantList.wantBlock.resolves({ + sender: peerId, + cid, + block: data + }) + + session = await createBitswapSession({ + ...components, + logger: defaultLogger() + }, { + root: cid, + queryConcurrency: 5, + minProviders: 1, + maxProviders: 3, + connectedPeers: [] + }) + + const p = session.want(cid) + + expect(components.wantList.wantBlock.called).to.be.true() + expect(components.wantList.wantBlock.getCall(0).args[1]?.peerId?.toString()).to.equal(peerId.toString()) + + await expect(p).to.eventually.deep.equal(data) + }) +}) diff --git a/packages/bitswap/test/stats.spec.ts b/packages/bitswap/test/stats.spec.ts new file mode 100644 index 00000000..bd895295 --- /dev/null +++ b/packages/bitswap/test/stats.spec.ts @@ -0,0 +1,104 @@ +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { Stats } from '../src/stats.js' +import type { MetricGroup, Metrics } from '@libp2p/interface' + +interface StubbedStatsComponents { + metrics: StubbedInstance +} + +describe('stats', () => { + let stats: Stats + let components: StubbedStatsComponents + let metricGroup: StubbedInstance + + beforeEach(() => { + components = { + metrics: stubInterface() + } + + metricGroup = stubInterface() + + // @ts-expect-error tsc does not select correct method overload sig + components.metrics.registerMetricGroup.returns(metricGroup) + + stats = new Stats(components) + }) + + it('should update global blocks received', () => { + stats.updateBlocksReceived(1) + + expect(metricGroup.increment.calledWith({ + global: 1 + })).to.be.true() + }) + + it('should update blocks received from a peer', async () => { + const peerId = await createEd25519PeerId() + + stats.updateBlocksReceived(1, peerId) + + expect(metricGroup.increment.calledWith({ + global: 1, + [peerId.toString()]: 1 + })).to.be.true() + }) + + it('should update global duplicate blocks received', () => { + stats.updateDuplicateBlocksReceived(1) + + expect(metricGroup.increment.calledWith({ + global: 1 + })).to.be.true() + }) + + it('should update duplicate blocks received from a peer', async () => { + const peerId = await createEd25519PeerId() + + stats.updateDuplicateBlocksReceived(1, peerId) + + expect(metricGroup.increment.calledWith({ + global: 1, + [peerId.toString()]: 1 + })).to.be.true() + }) + + it('should update global data received', () => { + stats.updateDataReceived(1) + + expect(metricGroup.increment.calledWith({ + global: 1 + })).to.be.true() + }) + + it('should update data received from a peer', async () => { + const peerId = await createEd25519PeerId() + + stats.updateDataReceived(1, peerId) + + expect(metricGroup.increment.calledWith({ + global: 1, + [peerId.toString()]: 1 + })).to.be.true() + }) + + it('should update global duplicate data received', () => { + stats.updateDuplicateDataReceived(1) + + expect(metricGroup.increment.calledWith({ + global: 1 + })).to.be.true() + }) + + it('should update duplicate data received from a peer', async () => { + const peerId = await createEd25519PeerId() + + stats.updateDuplicateDataReceived(1, peerId) + + expect(metricGroup.increment.calledWith({ + global: 1, + [peerId.toString()]: 1 + })).to.be.true() + }) +}) diff --git a/packages/bitswap/test/want-list.spec.ts b/packages/bitswap/test/want-list.spec.ts new file mode 100644 index 00000000..333fe6df --- /dev/null +++ b/packages/bitswap/test/want-list.spec.ts @@ -0,0 +1,106 @@ +import { matchPeerId } from '@libp2p/interface-compliance-tests/matchers' +import { defaultLogger } from '@libp2p/logger' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { type Network } from '../src/network.js' +import { WantType } from '../src/pb/message.js' +import { WantList } from '../src/want-list.js' + +interface StubbedWantListComponents { + network: StubbedInstance +} + +describe('wantlist', () => { + let wantList: WantList + let components: StubbedWantListComponents + + beforeEach(() => { + components = { + network: stubInterface() + } + + wantList = new WantList({ + ...components, + logger: defaultLogger() + }) + + wantList.start() + }) + + afterEach(() => { + if (wantList != null) { + wantList.stop() + } + }) + + it('should add peers to peer list on connect', async () => { + const peerId = await createEd25519PeerId() + + await wantList.peerConnected(peerId) + + expect(wantList.peers.has(peerId)).to.be.true() + }) + + it('should remove peers to peer list on disconnect', async () => { + const peerId = await createEd25519PeerId() + + await wantList.peerConnected(peerId) + + expect(wantList.peers.has(peerId)).to.be.true() + + wantList.peerDisconnected(peerId) + + expect(wantList.peers.has(peerId)).to.be.false() + }) + + it('should want blocks', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const peerId = await createEd25519PeerId() + + await wantList.peerConnected(peerId) + + components.network.sendMessage.withArgs(matchPeerId(peerId)) + + await expect(wantList.wantBlock(cid, { + signal: AbortSignal.timeout(500) + })).to.eventually.be.rejected + .with.property('code', 'ABORT_ERR') + + const sentToPeer = components.network.sendMessage.getCall(0).args[0] + expect(sentToPeer.toString()).equal(peerId.toString()) + + const sentMessage = components.network.sendMessage.getCall(0).args[1] + expect(sentMessage).to.have.nested.property('wantlist.full', false) + expect(sentMessage).to.have.deep.nested.property('wantlist.entries[0].cid', cid.bytes) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].wantType', WantType.WantBlock) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].cancel', false) + }) + + it('should not send session block wants to non-session peers', async () => { + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const sessionPeer = await createEd25519PeerId() + const nonSessionPeer = await createEd25519PeerId() + + await wantList.peerConnected(sessionPeer) + await wantList.peerConnected(nonSessionPeer) + + await expect(wantList.wantBlock(cid, { + peerId: sessionPeer, + signal: AbortSignal.timeout(500) + })).to.eventually.be.rejected + .with.property('code', 'ABORT_ERR') + + expect(components.network.sendMessage.callCount).to.equal(1) + + const sentToPeer = components.network.sendMessage.getCall(0).args[0] + expect(sentToPeer.toString()).equal(sessionPeer.toString()) + + const sentMessage = components.network.sendMessage.getCall(0).args[1] + expect(sentMessage).to.have.nested.property('wantlist.full', false) + expect(sentMessage).to.have.deep.nested.property('wantlist.entries[0].cid', cid.bytes) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].wantType', WantType.WantBlock) + expect(sentMessage).to.have.nested.property('wantlist.entries[0].cancel', false) + }) +}) diff --git a/packages/bitswap/tsconfig.json b/packages/bitswap/tsconfig.json new file mode 100644 index 00000000..4c0bdf77 --- /dev/null +++ b/packages/bitswap/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../interface" + } + ] +} diff --git a/packages/bitswap/typedoc.json b/packages/bitswap/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/bitswap/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/block-brokers/package.json b/packages/block-brokers/package.json index 9de7a474..59c0c7fe 100644 --- a/packages/block-brokers/package.json +++ b/packages/block-brokers/package.json @@ -53,13 +53,13 @@ "test:electron-main": "aegir test -t electron-main" }, "dependencies": { + "@helia/bitswap": "^0.0.0", "@helia/interface": "^4.1.0", "@libp2p/interface": "^1.1.4", "@libp2p/utils": "^5.2.6", "@multiformats/multiaddr-matcher": "^1.2.0", "@multiformats/multiaddr-to-uri": "^10.0.1", "interface-blockstore": "^5.2.10", - "ipfs-bitswap": "^20.0.2", "multiformats": "^13.1.0", "p-defer": "^4.0.0", "progress-events": "^1.0.0" diff --git a/packages/block-brokers/src/bitswap.ts b/packages/block-brokers/src/bitswap.ts index dfe79ff1..f2506ee4 100644 --- a/packages/block-brokers/src/bitswap.ts +++ b/packages/block-brokers/src/bitswap.ts @@ -1,8 +1,8 @@ -import { createBitswap } from 'ipfs-bitswap' -import type { BlockAnnounceOptions, BlockBroker, BlockRetrievalOptions } from '@helia/interface/blocks' -import type { Libp2p, Startable } from '@libp2p/interface' +import { createBitswap } from '@helia/bitswap' +import type { BitswapOptions, Bitswap, BitswapWantBlockProgressEvents, BitswapNotifyProgressEvents } from '@helia/bitswap' +import type { BlockAnnounceOptions, BlockBroker, BlockRetrievalOptions, CreateSessionOptions, Routing } from '@helia/interface' +import type { Libp2p, Startable, ComponentLogger } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' -import type { Bitswap, BitswapNotifyProgressEvents, BitswapOptions, BitswapWantBlockProgressEvents } from 'ipfs-bitswap' import type { CID } from 'multiformats/cid' import type { MultihashHasher } from 'multiformats/hashes/interface' @@ -10,6 +10,8 @@ interface BitswapComponents { libp2p: Libp2p blockstore: Blockstore hashers: Record + routing: Routing + logger: ComponentLogger } export interface BitswapInit extends BitswapOptions { @@ -21,9 +23,9 @@ class BitswapBlockBroker implements BlockBroker> => { let hasher: MultihashHasher | undefined @@ -63,12 +65,26 @@ class BitswapBlockBroker implements BlockBroker): Promise { - this.bitswap.notify(cid, block, options) + await this.bitswap.notify(cid, block, options) } async retrieve (cid: CID, options: BlockRetrievalOptions = {}): Promise { return this.bitswap.want(cid, options) } + + async createSession (root: CID, options?: CreateSessionOptions): Promise> { + const session = await this.bitswap.createSession(root, options) + + return { + announce: async (cid, block, options) => { + await this.bitswap.notify(cid, block, options) + }, + + retrieve: async (cid, options) => { + return session.want(cid, options) + } + } + } } /** diff --git a/packages/block-brokers/tsconfig.json b/packages/block-brokers/tsconfig.json index 4c0bdf77..26c90c06 100644 --- a/packages/block-brokers/tsconfig.json +++ b/packages/block-brokers/tsconfig.json @@ -8,6 +8,9 @@ "test" ], "references": [ + { + "path": "../bitswap" + }, { "path": "../interface" } diff --git a/packages/interop/src/unixfs-bitswap.spec.ts b/packages/interop/src/unixfs-bitswap.spec.ts index e25206a0..6c6cc283 100644 --- a/packages/interop/src/unixfs-bitswap.spec.ts +++ b/packages/interop/src/unixfs-bitswap.spec.ts @@ -52,13 +52,9 @@ describe('@helia/unixfs - bitswap', () => { const cid = await unixFs.addFile(candidate) - const output: Uint8Array[] = [] + const bytes = await toBuffer(kubo.api.cat(CID.parse(cid.toString()))) - for await (const b of kubo.api.cat(cid)) { - output.push(b) - } - - expect(toBuffer(output)).to.equalBytes(toBuffer(input)) + expect(bytes).to.equalBytes(toBuffer(input)) }) it('should add a large file to kubo and fetch it from helia', async () => {