From 8b5930c144caa2e873439100917539ddd15ac65a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 20 Sep 2024 15:47:37 +0100 Subject: [PATCH] feat(filecoin-api): allow custom hashing function to be passed to aggregate builder --- packages/filecoin-api/package.json | 2 +- packages/filecoin-api/src/aggregator/api.ts | 3 +- .../src/aggregator/buffer-reducing.js | 7 +--- .../filecoin-api/src/aggregator/events.js | 1 + pnpm-lock.yaml | 37 +++++++++++++++++-- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/packages/filecoin-api/package.json b/packages/filecoin-api/package.json index 54ffd53d6..9524bf02c 100644 --- a/packages/filecoin-api/package.json +++ b/packages/filecoin-api/package.json @@ -162,7 +162,7 @@ "@ucanto/transport": "^9.1.1", "@web3-storage/capabilities": "workspace:^", "@web3-storage/content-claims": "^5.0.0", - "@web3-storage/data-segment": "^4.0.0", + "@web3-storage/data-segment": "^5.2.0", "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0", "p-map": "^6.0.0" }, diff --git a/packages/filecoin-api/src/aggregator/api.ts b/packages/filecoin-api/src/aggregator/api.ts index 24264abb0..a12fce06e 100644 --- a/packages/filecoin-api/src/aggregator/api.ts +++ b/packages/filecoin-api/src/aggregator/api.ts @@ -1,6 +1,6 @@ import type { Signer, Principal, Link } from '@ucanto/interface' import { InclusionProof } from '@web3-storage/capabilities/types' -import { PieceLink } from '@web3-storage/data-segment' +import { PieceLink, API as DataSegmentAPI } from '@web3-storage/data-segment' import { AggregatorService, DealerService, @@ -327,6 +327,7 @@ export interface AggregateConfig { minAggregateSize: number minUtilizationFactor: number prependBufferedPieces?: BufferedPiece[] + hasher?: DataSegmentAPI.SyncMultihashHasher } // Enums diff --git a/packages/filecoin-api/src/aggregator/buffer-reducing.js b/packages/filecoin-api/src/aggregator/buffer-reducing.js index ab947fbed..5e06b54f1 100644 --- a/packages/filecoin-api/src/aggregator/buffer-reducing.js +++ b/packages/filecoin-api/src/aggregator/buffer-reducing.js @@ -152,11 +152,7 @@ export async function handleBufferReducingWithoutAggregate({ * Attempt to build an aggregate with buffered pieces within ranges. * * @param {BufferedPiece[]} bufferedPieces - * @param {object} config - * @param {number} config.maxAggregateSize - * @param {number} config.minAggregateSize - * @param {number} config.minUtilizationFactor - * @param {BufferedPiece[]} [config.prependBufferedPieces] + * @param {import('./api.js').AggregateConfig} config */ export function aggregatePieces(bufferedPieces, config) { // Guarantee buffered pieces total size is bigger than the minimum utilization @@ -175,6 +171,7 @@ export function aggregatePieces(bufferedPieces, config) { // Create builder with maximum size and try to fill it up const builder = Aggregate.createBuilder({ size: Aggregate.Size.from(config.maxAggregateSize), + hasher: config.hasher, }) // add pieces to an aggregate until there is no more space, or no more pieces diff --git a/packages/filecoin-api/src/aggregator/events.js b/packages/filecoin-api/src/aggregator/events.js index 24e96be39..d29b25b8c 100644 --- a/packages/filecoin-api/src/aggregator/events.js +++ b/packages/filecoin-api/src/aggregator/events.js @@ -115,6 +115,7 @@ export const handleBufferQueueMessage = async (context, records) => { minAggregateSize: context.config.minAggregateSize, minUtilizationFactor: context.config.minUtilizationFactor, prependBufferedPieces: context.config.prependBufferedPieces, + hasher: context.config.hasher, }) // Store buffered pieces if not enough to do aggregate and re-queue them diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a4c1f0367..164ae26d4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -307,8 +307,8 @@ importers: specifier: ^5.0.0 version: 5.0.0 '@web3-storage/data-segment': - specifier: ^4.0.0 - version: 4.0.0 + specifier: ^5.2.0 + version: 5.2.0 fr32-sha2-256-trunc254-padded-binary-tree-multihash: specifier: ^3.3.0 version: 3.3.0 @@ -1776,6 +1776,7 @@ packages: '@humanwhocodes/config-array@0.11.14': resolution: {integrity: sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==} engines: {node: '>=10.10.0'} + deprecated: Use @eslint/config-array instead '@humanwhocodes/module-importer@1.0.1': resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} @@ -1783,6 +1784,7 @@ packages: '@humanwhocodes/object-schema@2.0.3': resolution: {integrity: sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==} + deprecated: Use @eslint/object-schema instead '@ipld/car@5.3.0': resolution: {integrity: sha512-OB8LVvJeVAFFGluNIkZeDZ/aGeoekFKsuIvNT9I5sJIb5WekQuW5+lekjQ7Z7mZ7DBKuke/kI4jBT1j0/akU1w==} @@ -1792,6 +1794,10 @@ packages: resolution: {integrity: sha512-N14oMy0q4gM6OuZkIpisKe0JBSjf1Jb39VI+7jMLiWX9124u1Z3Fdj/Tag1NA0cVxxqWDh0CqsjcVfOKtelPDA==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} + '@ipld/dag-cbor@9.2.1': + resolution: {integrity: sha512-nyY48yE7r3dnJVlxrdaimrbloh4RokQaNRdI//btfTkcTEZbpmSrbYcBQ4VKTf8ZxXAOUJy4VsRpkJo+y9RTnA==} + engines: {node: '>=16.0.0', npm: '>=7.0.0'} + '@ipld/dag-json@10.2.0': resolution: {integrity: sha512-O9YLUrl3d3WbVz7v1WkajFkyfOLEe2Fep+wor4fgVe0ywxzrivrj437NiPcVyB+2EDdFn/Q7tCHFf8YVhDf8ZA==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -2377,6 +2383,9 @@ packages: '@web3-storage/data-segment@5.1.0': resolution: {integrity: sha512-FYdmtKvNiVz+maZ++k4PdD43rfJW5DeagLpstq2y84CyOKNRBWbHLCZ/Ec5zT9iGI+0WgsCGbpC/WlG0jlrnhA==} + '@web3-storage/data-segment@5.2.0': + resolution: {integrity: sha512-Jr/bdweoEQ0lWIaNuFcYxM6BHYmXHBWwfXygHyp370agkAdU+dIFIbm+tX+66XP/1YmEUi4JI2I80psDtoJ++Q==} + '@web3-storage/sigv4@1.0.2': resolution: {integrity: sha512-ZUXKK10NmuQgPkqByhb1H3OQxkIM0CIn2BMPhGQw7vQw8WIzrBkk9IJiAVfJ/UVBFrf6uzPbx2lEBLt4diCMnQ==} @@ -2977,7 +2986,7 @@ packages: engines: {node: '>= 0.8.0'} concat-map@0.0.1: - resolution: {integrity: sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=} + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} conf@11.0.2: resolution: {integrity: sha512-jjyhlQ0ew/iwmtwsS2RaB6s8DBifcE2GYBEaw2SJDUY/slJJbNfY4GlDVzOs/ff8cM/Wua5CikqXgbFl5eu85A==} @@ -3845,10 +3854,12 @@ packages: glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Glob versions prior to v9 are no longer supported glob@8.1.0: resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} engines: {node: '>=12'} + deprecated: Glob versions prior to v9 are no longer supported global-dirs@3.0.1: resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==} @@ -4157,6 +4168,7 @@ packages: inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. inherits@2.0.3: resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} @@ -5107,6 +5119,9 @@ packages: multiformats@13.1.0: resolution: {integrity: sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==} + multiformats@13.3.0: + resolution: {integrity: sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==} + multimatch@5.0.0: resolution: {integrity: sha512-ypMKuglUrZUD99Tk2bUQ+xNQj43lPEfAeX2o9cTteAmShXy2VHDJpuwu1o0xqoKCt9jLVAvwyFKdLTPXKAfJyA==} engines: {node: '>=10'} @@ -6079,6 +6094,7 @@ packages: rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true rtl-detect@1.1.2: @@ -8777,6 +8793,11 @@ snapshots: cborg: 4.2.0 multiformats: 13.1.0 + '@ipld/dag-cbor@9.2.1': + dependencies: + cborg: 4.2.0 + multiformats: 13.3.0 + '@ipld/dag-json@10.2.0': dependencies: cborg: 4.2.0 @@ -9522,7 +9543,7 @@ snapshots: '@web3-storage/data-segment@3.2.0': dependencies: - '@ipld/dag-cbor': 9.2.0 + '@ipld/dag-cbor': 9.2.1 multiformats: 11.0.2 sync-multihash-sha2: 1.0.0 @@ -9538,6 +9559,12 @@ snapshots: multiformats: 11.0.2 sync-multihash-sha2: 1.0.0 + '@web3-storage/data-segment@5.2.0': + dependencies: + '@ipld/dag-cbor': 9.2.1 + multiformats: 13.3.0 + sync-multihash-sha2: 1.0.0 + '@web3-storage/sigv4@1.0.2': dependencies: '@noble/hashes': 1.4.0 @@ -12953,6 +12980,8 @@ snapshots: multiformats@13.1.0: {} + multiformats@13.3.0: {} + multimatch@5.0.0: dependencies: '@types/minimatch': 3.0.5