Skip to content

Commit

Permalink
fix!: update aggregation capabilitites to use height instead of size …
Browse files Browse the repository at this point in the history
…together with client and api (#831)

Renames `size` to `height` for `piece`/`aggregate` based on described in
storacha/data-segment#13

Note that:
- took the opportunity to update this to finally rely on `data-segment`
lib as it is now ready to be used (both in tests, and to validate
aggregate)
- given `piece` and `aggregate` are both perfect binary trees, leaf
count of can be derived as `2 ** height`, and the size of the tree can
be derived by `leafCount * Node.Size` (32).

Needs storacha/specs#66
  • Loading branch information
vasco-santos authored Jul 25, 2023
1 parent 338615b commit 31730f0
Show file tree
Hide file tree
Showing 16 changed files with 1,163 additions and 1,104 deletions.
2 changes: 1 addition & 1 deletion packages/aggregate-api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
"@ucanto/server": "^8.0.0",
"@ucanto/transport": "^8.0.0",
"@web3-storage/capabilities": "workspace:^",
"@web3-storage/data-segment": "^1.0.1"
"@web3-storage/data-segment": "^2.2.0"
},
"devDependencies": {
"@ipld/car": "^5.1.1",
Expand Down
43 changes: 31 additions & 12 deletions packages/aggregate-api/src/aggregate/offer.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import * as Server from '@ucanto/server'
import { CBOR } from '@ucanto/core'
import { Node, Piece, Aggregate as AggregateBuilder } from '@web3-storage/data-segment'
import * as Aggregate from '@web3-storage/capabilities/aggregate'
import * as Offer from '@web3-storage/capabilities/offer'
import * as API from '../types.js'

export const MIN_SIZE = 1 + 127 * (1 << 27)
export const MAX_SIZE = 127 * (1 << 28)
// 16 GiB
export const MIN_SIZE = Piece.PaddedSize.from(2n ** 34n)
// 32 GiB
export const MAX_SIZE = Piece.PaddedSize.from(2n ** 35n)

/**
* @param {API.AggregateServiceContext} context
Expand Down Expand Up @@ -39,29 +42,45 @@ export const claim = async (
}

// Validate offer content
const size = offers.reduce((accum, offer) => accum + offer.size, 0)
if (size < MIN_SIZE) {
const aggregateLeafs = 2n ** BigInt(piece.height)
const aggregateSize = aggregateLeafs * BigInt(Node.Size)

if (aggregateSize < MIN_SIZE) {
return {
error: new AggregateOfferInvalidSizeError(
`offer under size, offered: ${size}, minimum: ${MIN_SIZE}`
`offer under size, offered: ${aggregateSize}, minimum: ${MIN_SIZE}`
),
}
} else if (size > MAX_SIZE) {
} else if (aggregateSize > MAX_SIZE) {
return {
error: new AggregateOfferInvalidSizeError(
`offer over size, offered: ${size}, maximum: ${MAX_SIZE}`
`offer over size, offered: ${aggregateSize}, maximum: ${MAX_SIZE}`
),
}
} else if (size !== piece.size) {
}

// Validate commP of commPs
const aggregateBuild = AggregateBuilder.build({
size: aggregateSize,
pieces: offers.map(offer => Piece.fromJSON({
height: offer.height,
link: { '/': offer.link.toString() }
}))
})
if (!aggregateBuild.link.equals(piece.link)) {
return {
error: new AggregateOfferInvalidSizeError(
`offer size mismatch, specified: ${piece.size}, actual: ${size}`
`aggregate piece CID mismatch, specified: ${piece.link}, computed: ${aggregateBuild.link}`
),
}
} else if (aggregateBuild.height !== piece.height) {
return {
error: new AggregateOfferInvalidSizeError(
`aggregate height mismatch, specified: ${piece.height}, computed: ${aggregateBuild.height}`
),
}
}

// TODO: Validate commP of commPs

// Create effect for receipt
const fx = await Offer.arrange
.invoke({
Expand Down Expand Up @@ -90,7 +109,7 @@ function getOfferBlock(offerCid, blockIterator) {
for (const block of blockIterator) {
if (block.cid.equals(offerCid)) {
const decoded =
/** @type {import('@web3-storage/aggregate-client/types').Piece[]} */ (
/** @type {import('@web3-storage/data-segment').PieceView[]} */ (
CBOR.decode(block.bytes)
)
return decoded
Expand Down
14 changes: 5 additions & 9 deletions packages/aggregate-api/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import type {
} from '@ucanto/interface'
import type { ProviderInput } from '@ucanto/server'

import type { Piece } from '@web3-storage/aggregate-client/types'
import type { PieceLink, PieceView } from '@web3-storage/data-segment'
export * from '@web3-storage/aggregate-client/types'

export * from '@web3-storage/capabilities/types'
Expand All @@ -30,24 +30,20 @@ export interface ServiceContext
OfferServiceContext {}

export interface ArrangedOfferStore {
get: (
pieceLink: Link<unknown, number, number, 0 | 1>
) => Promise<string | undefined>
get: (pieceLink: PieceLink) => Promise<string | undefined>
}

export interface OfferStore {
queue: (aggregateOffer: OfferToQueue) => Promise<void>
}

export interface OfferToQueue {
piece: Piece
offers: Piece[]
piece: PieceView
offers: PieceView[]
}

export interface AggregateStore {
get: (
pieceLink: Link<unknown, number, number, 0 | 1>
) => Promise<unknown[] | undefined>
get: (pieceLink: PieceLink) => Promise<unknown[] | undefined>
}

export interface UcantoServerContext extends ServiceContext {
Expand Down
Loading

0 comments on commit 31730f0

Please sign in to comment.