From 1a456e8ab1bc0b1e2381ee3c55ad45a195f7948e Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 25 Mar 2024 11:33:44 +0100 Subject: [PATCH 01/27] feat: blob implementation --- packages/capabilities/package.json | 3 +- packages/capabilities/src/blob.js | 228 +++++++ packages/capabilities/src/index.js | 8 + packages/capabilities/src/types.ts | 84 ++- packages/capabilities/src/utils.js | 61 ++ packages/upload-api/package.json | 1 + packages/upload-api/src/blob.js | 15 + packages/upload-api/src/blob/accept.js | 27 + packages/upload-api/src/blob/add.js | 78 +++ packages/upload-api/src/blob/allocate.js | 100 +++ packages/upload-api/src/blob/lib.js | 56 ++ packages/upload-api/src/blob/list.js | 15 + packages/upload-api/src/blob/remove.js | 22 + packages/upload-api/src/lib.js | 4 + packages/upload-api/src/service.js | 15 + packages/upload-api/src/types.ts | 112 +++- packages/upload-api/test/handlers/blob.js | 612 ++++++++++++++++++ .../upload-api/test/handlers/blob.spec.js | 4 + packages/upload-api/test/helpers/context.js | 21 +- packages/upload-api/test/lib.js | 3 + .../test/storage/allocation-storage.js | 109 ++++ .../upload-api/test/storage/blob-storage.js | 169 +++++ pnpm-lock.yaml | 15 + 23 files changed, 1754 insertions(+), 8 deletions(-) create mode 100644 packages/capabilities/src/blob.js create mode 100644 packages/upload-api/src/blob.js create mode 100644 packages/upload-api/src/blob/accept.js create mode 100644 packages/upload-api/src/blob/add.js create mode 100644 packages/upload-api/src/blob/allocate.js create mode 100644 packages/upload-api/src/blob/lib.js create mode 100644 packages/upload-api/src/blob/list.js create mode 100644 packages/upload-api/src/blob/remove.js create mode 100644 packages/upload-api/src/service.js create mode 100644 packages/upload-api/test/handlers/blob.js create mode 100644 packages/upload-api/test/handlers/blob.spec.js create mode 100644 packages/upload-api/test/storage/allocation-storage.js create mode 100644 packages/upload-api/test/storage/blob-storage.js diff --git a/packages/capabilities/package.json b/packages/capabilities/package.json index 5fcd87bd2..34bc7711e 100644 --- a/packages/capabilities/package.json +++ b/packages/capabilities/package.json @@ -88,7 +88,8 @@ "@ucanto/principal": "^9.0.1", "@ucanto/transport": "^9.1.1", "@ucanto/validator": "^9.0.2", - "@web3-storage/data-segment": "^3.2.0" + "@web3-storage/data-segment": "^3.2.0", + "uint8arrays": "^5.0.3" }, "devDependencies": { "@web3-storage/eslint-config-w3up": "workspace:^", diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js new file mode 100644 index 000000000..3a96a7e3e --- /dev/null +++ b/packages/capabilities/src/blob.js @@ -0,0 +1,228 @@ +/** + * Blob Capabilities. + * + * Blob is a fixed size byte array addressed by the multihash. + * Usually blobs are used to represent set of IPLD blocks at different byte ranges. + * + * These can be imported directly with: + * ```js + * import * as Blob from '@web3-storage/capabilities/blob' + * ``` + * + * @module + */ +import { capability, Link, Schema, ok, fail } from '@ucanto/validator' +import { + equal, + equalBlob, + equalContent, + equalWith, + checkLink, + SpaceDID, + and, +} from './utils.js' + +/** + * Agent capabilities for Blob protocol + */ + +/** + * Capability can only be delegated (but not invoked) allowing audience to + * derived any `blob/` prefixed capability for the (memory) space identified + * by DID in the `with` field. + */ +export const blob = capability({ + can: 'blob/*', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + derives: equalWith, +}) + +/** + * Blob description for being ingested by the service. + */ +export const blobStruct = Schema.struct({ + /** + * A multihash digest of the blob payload bytes, uniquely identifying blob. + */ + content: Schema.bytes(), + /** + * Size of the Blob file to be stored. Service will provision write target + * for this exact size. Attempt to write a larger Blob file will fail. + */ + size: Schema.integer(), +}) + +/** + * `blob/add` capability allows agent to store a Blob into a (memory) space + * identified by did:key in the `with` field. Agent must precompute Blob locally + * and provide it's multihash and size using `nb.content` and `nb.size` fields, allowing + * a service to provision a write location for the agent to PUT or POST desired + * Blob into. + */ +export const add = capability({ + can: 'blob/add', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * Blob to allocate on the space. + */ + blob: blobStruct, + }), + derives: equalBlob, +}) + +/** + * `blob/remove` capability can be used to remove the stored Blob from the (memory) + * space identified by `with` field. + */ +export const remove = capability({ + can: 'blob/remove', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * A multihash digest of the blob payload bytes, uniquely identifying blob. + */ + content: Schema.bytes(), + }), + derives: equalContent, +}) + +/** + * `blob/list` capability can be invoked to request a list of stored Blobs in the + * (memory) space identified by `with` field. + */ +export const list = capability({ + can: 'blob/list', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * A pointer that can be moved back and forth on the list. + * It can be used to paginate a list for instance. + */ + cursor: Schema.string().optional(), + /** + * Maximum number of items per page. + */ + size: Schema.integer().optional(), + /** + * If true, return page of results preceding cursor. Defaults to false. + */ + pre: Schema.boolean().optional(), + }), + derives: (claimed, delegated) => { + if (claimed.with !== delegated.with) { + return fail( + `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` + ) + } + return ok({}) + }, +}) + +/** + * Service capabilities for Blob protocol + */ +/** + * Capability can only be delegated (but not invoked) allowing audience to + * derived any `web3.storage/blob/` prefixed capability for the (memory) space identified + * by DID in the `with` field. + */ +export const serviceBlob = capability({ + can: 'web3.storage/blob/*', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + derives: equalWith, +}) + +/** + * `web3.storage/blob//allocate` capability can be invoked to create a memory + * address where blob content can be written via HTTP PUT request. + */ +export const allocate = capability({ + can: 'web3.storage/blob/allocate', + /** + * Provider DID. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * Blob to allocate on the space. + */ + blob: blobStruct, + /** + * The Link for an Add Blob task, that caused an allocation + */ + cause: Link, + /** + * DID of the user space where allocation takes place + */ + space: SpaceDID, + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(equalBlob(claim, from)) || + and(checkLink(claim.nb.cause, from.nb.cause, 'cause')) || + and(equal(claim.nb.space, from.nb.space, 'space')) || + ok({}) + ) + }, +}) + +/** + * `blob/accept` capability invocation should either succeed when content is + * delivered on allocated address or fail if no content is allocation expires + * without content being delivered. + */ +export const accept = capability({ + can: 'web3.storage/blob/accept', + /** + * Provider DID. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * Blob to accept. + */ + blob: blobStruct, + /** + * Expiration.. + */ + exp: Schema.integer(), + }), + derives: (claim, from) => { + const result = equalBlob(claim, from) + if (result.error) { + return result + } else if (claim.nb.exp !== undefined && from.nb.exp !== undefined) { + return claim.nb.exp > from.nb.exp + ? fail(`exp constraint violation: ${claim.nb.exp} > ${from.nb.exp}`) + : ok({}) + } else { + return ok({}) + } + }, +}) + +// ⚠️ We export imports here so they are not omitted in generated typedes +// @see https://github.com/microsoft/TypeScript/issues/51548 +export { Schema, Link } diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index d80fbff46..8c423bfe1 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -19,6 +19,7 @@ import * as DealTracker from './filecoin/deal-tracker.js' import * as UCAN from './ucan.js' import * as Plan from './plan.js' import * as Usage from './usage.js' +import * as Blob from './blob.js' export { Access, @@ -86,4 +87,11 @@ export const abilitiesAsStrings = [ Plan.get.can, Usage.usage.can, Usage.report.can, + Blob.blob.can, + Blob.add.can, + Blob.remove.can, + Blob.list.can, + Blob.serviceBlob.can, + Blob.allocate.can, + Blob.accept.can, ] diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 9848a42ca..9944af005 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -21,6 +21,7 @@ import { import { space, info } from './space.js' import * as provider from './provider.js' import { top } from './top.js' +import * as BlobCaps from './blob.js' import * as StoreCaps from './store.js' import * as UploadCaps from './upload.js' import * as AccessCaps from './access.js' @@ -439,6 +440,80 @@ export interface UploadNotFound extends Ucanto.Failure { export type UploadGetFailure = UploadNotFound | Ucanto.Failure +// Blob +export type Blob = InferInvokedCapability +export type BlobAdd = InferInvokedCapability +export type BlobRemove = InferInvokedCapability +export type BlobList = InferInvokedCapability +export type ServiceBlob = InferInvokedCapability +export type BlobAllocate = InferInvokedCapability +export type BlobAccept = InferInvokedCapability + +export type BlobMultihash = Uint8Array + +// Blob add +export interface BlobAddSuccess { + claim: { + 'await/ok': Link + } +} + +export interface BlobItemSizeExceeded extends Ucanto.Failure { + name: 'BlobItemSizeExceeded' +} +export type BlobAddFailure = BlobItemSizeExceeded | Ucanto.Failure + +// Blob remove +export interface BlobRemoveSuccess { + size: number +} + +export interface BlobItemNotFound extends Ucanto.Failure { + name: 'BlobItemNotFound' +} + +export type BlobRemoveFailure = BlobItemNotFound | Ucanto.Failure + +// Blob list +export interface BlobListSuccess extends ListResponse {} +export interface BlobListItem { + blob: { content: Uint8Array; size: number } + insertedAt: ISO8601Date +} + +export type BlobListFailure = Ucanto.Failure + +// Blob allocate +export interface BlobAllocateSuccess { + size: number + address?: BlobAddress +} + +export interface BlobAddress { + url: ToString + headers: Record +} + +export interface BlobItemNotFound extends Ucanto.Failure { + name: 'BlobItemNotFound' +} + +export interface BlobNotAllocableToSpace extends Ucanto.Failure { + name: 'BlobNotAllocableToSpace' +} + +export type BlobAllocateFailure = + | BlobItemNotFound + | BlobNotAllocableToSpace + | Ucanto.Failure + +// Blob accept +export interface BlobAcceptSuccess { + claim: Link +} + +export type BlobAcceptFailure = BlobItemNotFound | Ucanto.Failure + // Store export type Store = InferInvokedCapability export type StoreAdd = InferInvokedCapability @@ -708,7 +783,14 @@ export type ServiceAbilityArray = [ AdminStoreInspect['can'], PlanGet['can'], Usage['can'], - UsageReport['can'] + UsageReport['can'], + Blob['can'], + BlobAdd['can'], + BlobRemove['can'], + BlobList['can'], + ServiceBlob['can'], + BlobAllocate['can'], + BlobAccept['can'] ] /** diff --git a/packages/capabilities/src/utils.js b/packages/capabilities/src/utils.js index ac1e7e317..96c63c352 100644 --- a/packages/capabilities/src/utils.js +++ b/packages/capabilities/src/utils.js @@ -2,6 +2,8 @@ import { DID, fail, ok } from '@ucanto/validator' // eslint-disable-next-line no-unused-vars import * as Types from '@ucanto/interface' +import { equals } from 'uint8arrays/equals' + // e.g. did:web:web3.storage or did:web:staging.web3.storage export const ProviderDID = DID.match({ method: 'web' }) @@ -85,6 +87,65 @@ export const equalLink = (claimed, delegated) => { } } +/** + * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"web3.storage/blob/allocate"|"web3.storage/blob/accept", Types.URI<'did:'>, {blob: { content: Uint8Array, size: number }}>} T + * @param {T} claimed + * @param {T} delegated + * @returns {Types.Result<{}, Types.Failure>} + */ +export const equalBlob = (claimed, delegated) => { + if (claimed.with !== delegated.with) { + return fail( + `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` + ) + } else if ( + delegated.nb.blob.content && + !equals(delegated.nb.blob.content, claimed.nb.blob.content) + ) { + return fail( + `Link ${ + claimed.nb.blob.content ? `${claimed.nb.blob.content}` : '' + } violates imposed ${delegated.nb.blob.content} constraint.` + ) + } else if ( + claimed.nb.blob.size !== undefined && + delegated.nb.blob.size !== undefined + ) { + return claimed.nb.blob.size > delegated.nb.blob.size + ? fail( + `Size constraint violation: ${claimed.nb.blob.size} > ${delegated.nb.blob.size}` + ) + : ok({}) + } else { + return ok({}) + } +} + +/** + * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"blob/allocate"|"blob/accept", Types.URI<'did:'>, {content: Uint8Array}>} T + * @param {T} claimed + * @param {T} delegated + * @returns {Types.Result<{}, Types.Failure>} + */ +export const equalContent = (claimed, delegated) => { + if (claimed.with !== delegated.with) { + return fail( + `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` + ) + } else if ( + delegated.nb.content && + !equals(delegated.nb.content, claimed.nb.content) + ) { + return fail( + `Link ${ + claimed.nb.content ? `${claimed.nb.content}` : '' + } violates imposed ${delegated.nb.content} constraint.` + ) + } else { + return ok({}) + } +} + /** * Checks that `claimed` {@link Types.Link} meets an `imposed` constraint. * diff --git a/packages/upload-api/package.json b/packages/upload-api/package.json index 99cff513d..6c44985df 100644 --- a/packages/upload-api/package.json +++ b/packages/upload-api/package.json @@ -182,6 +182,7 @@ "@web3-storage/did-mailto": "workspace:^", "@web3-storage/filecoin-api": "workspace:^", "multiformats": "^12.1.2", + "uint8arrays": "^5.0.3", "p-retry": "^5.1.2" }, "devDependencies": { diff --git a/packages/upload-api/src/blob.js b/packages/upload-api/src/blob.js new file mode 100644 index 000000000..78b4bb40b --- /dev/null +++ b/packages/upload-api/src/blob.js @@ -0,0 +1,15 @@ +import { blobAddProvider } from './blob/add.js' +import { blobListProvider } from './blob/list.js' +import { blobRemoveProvider } from './blob/remove.js' +import * as API from './types.js' + +/** + * @param {API.BlobServiceContext} context + */ +export function createService(context) { + return { + add: blobAddProvider(context), + list: blobListProvider(context), + remove: blobRemoveProvider(context), + } +} diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js new file mode 100644 index 000000000..6c86e7e1d --- /dev/null +++ b/packages/upload-api/src/blob/accept.js @@ -0,0 +1,27 @@ +import * as Server from '@ucanto/server' +import * as Blob from '@web3-storage/capabilities/blob' +import * as API from '../types.js' +import { BlobItemNotFound } from './lib.js' + +/** + * @param {API.W3ServiceContext} context + * @returns {API.ServiceMethod} + */ +export function blobAcceptProvider(context) { + return Server.provide(Blob.accept, async ({ capability }) => { + const { blob } = capability.nb + // If blob is not stored, we must fail + const hasBlob = await context.blobStorage.has(blob.content) + if (hasBlob.error) { + return { + error: new BlobItemNotFound(), + } + } + + // TODO: return content commitment + + return { + error: new BlobItemNotFound(), + } + }) +} diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js new file mode 100644 index 000000000..3c9825a36 --- /dev/null +++ b/packages/upload-api/src/blob/add.js @@ -0,0 +1,78 @@ +import * as Server from '@ucanto/server' +import * as Blob from '@web3-storage/capabilities/blob' +import * as API from '../types.js' + +import { BlobItemSizeExceeded } from './lib.js' + +/** + * @param {API.BlobServiceContext} context + * @returns {API.ServiceMethod} + */ +export function blobAddProvider(context) { + return Server.provideAdvanced({ + capability: Blob.add, + handler: async ({ capability, invocation }) => { + const { id, allocationStorage, maxUploadSize, getServiceConnection } = context + const { blob } = capability.nb + const space = /** @type {import('@ucanto/interface').DIDKey} */ ( + Server.DID.parse(capability.with).did() + ) + + if (blob.size > maxUploadSize) { + return { + error: new BlobItemSizeExceeded(maxUploadSize) + } + } + + // Create effects for receipt + // TODO: needs HTTP/PUT receipt + const blobAllocate = Blob.allocate + .invoke({ + issuer: id, + audience: id, + with: id.did(), + nb: { + blob, + cause: invocation.link(), + space, + }, + expiration: Infinity + }) + const blobAccept = Blob.accept + .invoke({ + issuer: id, + audience: id, + with: id.toDIDKey(), + nb: { + blob, + exp: Number.MAX_SAFE_INTEGER, + }, + expiration: Infinity, + }) + const [allocatefx, acceptfx] = await Promise.all([ + blobAllocate.delegate(), + blobAccept.delegate(), + ]) + + // Schedule allocation if not allocated + const allocated = await allocationStorage.exists(space, blob.content) + if (!allocated.ok) { + // Execute allocate invocation + const allocateRes = await blobAllocate.execute(getServiceConnection()) + if (allocateRes.out.error) { + return { + error: allocateRes.out.error + } + } + } + + /** @type {API.OkBuilder} */ + const result = Server.ok({ + claim: { + 'await/ok': acceptfx.link(), + }, + }) + return result.fork(allocatefx.link()).join(acceptfx.link()) + }, + }) +} diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js new file mode 100644 index 000000000..0244f80e3 --- /dev/null +++ b/packages/upload-api/src/blob/allocate.js @@ -0,0 +1,100 @@ +import * as Server from '@ucanto/server' +import * as Blob from '@web3-storage/capabilities/blob' +import * as API from '../types.js' +import { BlobItemNotFound } from './lib.js' +import { ensureRateLimitAbove } from '../utils/rate-limits.js' + +/** + * @param {API.W3ServiceContext} context + * @returns {API.ServiceMethod} + */ +export function blobAllocateProvider(context) { + return Server.provide(Blob.allocate, async ({ capability, invocation }) => { + const { blob, cause, space } = capability.nb + + // Rate limiting validation + const rateLimitResult = await ensureRateLimitAbove( + context.rateLimitsStorage, + [space], + 0 + ) + if (rateLimitResult.error) { + return { + error: { + name: 'InsufficientStorage', + message: `${space} is blocked`, + }, + } + } + + // Has Storage provider validation + const result = await context.provisionsStorage.hasStorageProvider(space) + if (result.error) { + return result + } + if (!result.ok) { + return { + /** @type {API.AllocationError} */ + error: { + name: 'InsufficientStorage', + message: `${space} has no storage provider`, + }, + } + } + + // If blob is stored, we can just allocate it to the space + const hasBlob = await context.blobStorage.has(blob.content) + if (hasBlob.error) { + return { + error: new BlobItemNotFound(space), + } + } + // Get presigned URL for the write target + const createUploadUrl = await context.blobStorage.createUploadUrl( + blob.content, + blob.size + ) + if (createUploadUrl.error) { + return { + error: new Server.Failure('failed to provide presigned url'), + } + } + + // Allocate in space, ignoring if already allocated + const allocationInsert = await context.allocationStorage.insert({ + space, + blob, + invocation: cause, + // TODO: add write target here + // will the URL be enough to track? + }) + if (allocationInsert.error) { + // if the insert failed with conflict then this item has already been + // added to the space and there is no allocation change. + if (allocationInsert.error.name === 'RecordKeyConflict') { + return { + ok: { size: 0 }, + } + } + return { + error: new Server.Failure('failed to allocate blob bytes'), + } + } + + if (hasBlob.ok) { + return { + ok: { size: blob.size }, + } + } + + return { + ok: { + size: blob.size, + address: { + url: createUploadUrl.ok.url.toString(), + headers: createUploadUrl.ok.headers, + }, + }, + } + }) +} diff --git a/packages/upload-api/src/blob/lib.js b/packages/upload-api/src/blob/lib.js new file mode 100644 index 000000000..ff3995109 --- /dev/null +++ b/packages/upload-api/src/blob/lib.js @@ -0,0 +1,56 @@ +import { Failure } from '@ucanto/server' + +export const BlobItemNotFoundName = 'BlobItemNotFound' +export class BlobItemNotFound extends Failure { + /** + * @param {import('@ucanto/interface').DID} [space] + */ + constructor(space) { + super() + this.space = space + } + + get name() { + return BlobItemNotFoundName + } + + describe() { + if (this.space) { + return `Blob not found in ${this.space}` + } + return `Blob not found` + } + + toJSON() { + return { + ...super.toJSON(), + space: this.space, + } + } +} + +export const BlobItemSizeExceededName = 'BlobItemSizeExceeded' +export class BlobItemSizeExceeded extends Failure { + /** + * @param {Number} maxUploadSize + */ + constructor(maxUploadSize) { + super() + this.maxUploadSize = maxUploadSize + } + + get name() { + return BlobItemSizeExceededName + } + + describe() { + return `Maximum size exceeded: ${this.maxUploadSize}, split DAG into smaller shards.` + } + + toJSON() { + return { + ...super.toJSON(), + maxUploadSize: this.maxUploadSize, + } + } +} diff --git a/packages/upload-api/src/blob/list.js b/packages/upload-api/src/blob/list.js new file mode 100644 index 000000000..6694804fd --- /dev/null +++ b/packages/upload-api/src/blob/list.js @@ -0,0 +1,15 @@ +import * as Server from '@ucanto/server' +import * as Blob from '@web3-storage/capabilities/blob' +import * as API from '../types.js' + +/** + * @param {API.BlobServiceContext} context + * @returns {API.ServiceMethod} + */ +export function blobListProvider(context) { + return Server.provide(Blob.list, async ({ capability }) => { + const { cursor, size, pre } = capability.nb + const space = Server.DID.parse(capability.with).did() + return await context.allocationStorage.list(space, { size, cursor, pre }) + }) +} diff --git a/packages/upload-api/src/blob/remove.js b/packages/upload-api/src/blob/remove.js new file mode 100644 index 000000000..546f4935f --- /dev/null +++ b/packages/upload-api/src/blob/remove.js @@ -0,0 +1,22 @@ +import * as Server from '@ucanto/server' +import * as Blob from '@web3-storage/capabilities/blob' +import * as API from '../types.js' +import { BlobItemNotFound } from './lib.js' + +/** + * @param {API.BlobServiceContext} context + * @returns {API.ServiceMethod} + */ +export function blobRemoveProvider(context) { + return Server.provide(Blob.remove, async ({ capability }) => { + const { content } = capability.nb + const space = Server.DID.parse(capability.with).did() + + const res = await context.allocationStorage.remove(space, content) + if (res.error && res.error.name === 'RecordNotFound') { + return Server.error(new BlobItemNotFound(space)) + } + + return res + }) +} diff --git a/packages/upload-api/src/lib.js b/packages/upload-api/src/lib.js index c4fd4ebaa..03ba7184b 100644 --- a/packages/upload-api/src/lib.js +++ b/packages/upload-api/src/lib.js @@ -4,6 +4,7 @@ import * as Types from './types.js' import * as Legacy from '@ucanto/transport/legacy' import * as CAR from '@ucanto/transport/car' import { create as createRevocationChecker } from './utils/revocation.js' +import { createService as createBlobService } from './blob.js' import { createService as createStoreService } from './store.js' import { createService as createUploadService } from './upload.js' import { createService as createConsoleService } from './console.js' @@ -16,6 +17,7 @@ import { createService as createSubscriptionService } from './subscription.js' import { createService as createAdminService } from './admin.js' import { createService as createRateLimitService } from './rate-limit.js' import { createService as createUcanService } from './ucan.js' +import { createService as createW3sService } from './service.js' import { createService as createPlanService } from './plan.js' import { createService as createUsageService } from './usage.js' import { createService as createFilecoinService } from '@web3-storage/filecoin-api/storefront/service' @@ -43,6 +45,7 @@ export const createServer = ({ id, codec = Legacy.inbound, ...context }) => */ export const createService = (context) => ({ access: createAccessService(context), + blob: createBlobService(context), console: createConsoleService(context), consumer: createConsumerService(context), customer: createCustomerService(context), @@ -55,6 +58,7 @@ export const createService = (context) => ({ upload: createUploadService(context), ucan: createUcanService(context), plan: createPlanService(context), + ['web3.storage']: createW3sService(context), // storefront of filecoin pipeline filecoin: createFilecoinService(context).filecoin, usage: createUsageService(context), diff --git a/packages/upload-api/src/service.js b/packages/upload-api/src/service.js new file mode 100644 index 000000000..bb9c503bc --- /dev/null +++ b/packages/upload-api/src/service.js @@ -0,0 +1,15 @@ +import { blobAllocateProvider } from './blob/allocate.js' +import { blobAcceptProvider } from './blob/accept.js' +import * as API from './types.js' + +/** + * @param {API.W3ServiceContext} context + */ +export function createService(context) { + return { + blob: { + allocate: blobAllocateProvider(context), + accept: blobAcceptProvider(context), + } + } +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 789be64e1..073d1a972 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -54,6 +54,23 @@ export interface DebugEmail extends Email { } import { + BlobMultihash, + BlobAdd, + BlobAddSuccess, + BlobAddFailure, + BlobRemove, + BlobRemoveSuccess, + BlobRemoveFailure, + BlobList, + BlobListItem, + BlobListSuccess, + BlobListFailure, + BlobAllocate, + BlobAllocateSuccess, + BlobAllocateFailure, + BlobAccept, + BlobAcceptSuccess, + BlobAcceptFailure, StoreAdd, StoreGet, StoreAddSuccess, @@ -162,7 +179,12 @@ export type { SubscriptionsStorage } import { UsageStorage } from './types/usage.js' export type { UsageStorage } -export interface Service extends StorefrontService { +export interface Service extends StorefrontService, W3sService { + blob: { + add: ServiceMethod + remove: ServiceMethod + list: ServiceMethod + } store: { add: ServiceMethod get: ServiceMethod @@ -273,9 +295,41 @@ export interface Service extends StorefrontService { } } -export type StoreServiceContext = SpaceServiceContext & { +export interface W3sService { + ['web3.storage']: { + blob: { + allocate: ServiceMethod< + BlobAllocate, + BlobAllocateSuccess, + BlobAllocateFailure + > + accept: ServiceMethod + } + } +} + +export type BlobServiceContext = SpaceServiceContext & { + /** + * Service signer + */ + id: Signer maxUploadSize: number + allocationStorage: AllocationStorage + blobStorage: BlobStorage + getServiceConnection: () => ConnectionView +} +export type W3ServiceContext = SpaceServiceContext & { + /** + * Service signer + */ + id: Signer + allocationStorage: AllocationStorage + blobStorage: BlobStorage +} + +export type StoreServiceContext = SpaceServiceContext & { + maxUploadSize: number storeTable: StoreTable carStoreBucket: CarStoreBucket } @@ -362,6 +416,7 @@ export interface ServiceContext ProviderServiceContext, SpaceServiceContext, StoreServiceContext, + BlobServiceContext, SubscriptionServiceContext, RateLimitServiceContext, RevocationServiceContext, @@ -396,6 +451,25 @@ export interface ErrorReporter { catch: (error: HandlerExecutionError) => void } +export interface BlobStorage { + has: (content: BlobMultihash) => Promise> + createUploadUrl: ( + content: BlobMultihash, + size: number + ) => Promise< + Result< + { + url: URL + headers: { + 'x-amz-checksum-sha256': string + 'content-length': string + } & Record + }, + Failure + > + > +} + export interface CarStoreBucket { has: (link: UnknownLink) => Promise createUploadUrl: ( @@ -442,6 +516,26 @@ export interface RecordKeyConflict extends Failure { name: 'RecordKeyConflict' } +export interface AllocationStorage { + exists: ( + space: DID, + blobMultihash: BlobMultihash + ) => Promise> + /** Inserts an item in the table if it does not already exist. */ + insert: ( + item: BlobAddInput + ) => Promise> + /** Removes an item from the table but fails if the item does not exist. */ + remove: ( + space: DID, + blobMultihash: BlobMultihash + ) => Promise> + list: ( + space: DID, + options?: ListOptions + ) => Promise, Failure>> +} + export interface StoreTable { inspect: (link: UnknownLink) => Promise> exists: (space: DID, link: UnknownLink) => Promise> @@ -510,6 +604,20 @@ export type AdminUploadInspectResult = Result< AdminUploadInspectFailure > +export interface Blob { + content: BlobMultihash + size: number +} + +export interface BlobAddInput { + space: DID + invocation: UnknownLink + blob: Blob +} + +export interface BlobAddOutput + extends Omit {} + export interface StoreAddInput { space: DID link: UnknownLink diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js new file mode 100644 index 000000000..acee6a788 --- /dev/null +++ b/packages/upload-api/test/handlers/blob.js @@ -0,0 +1,612 @@ +import * as API from '../../src/types.js' +import { Absentee } from '@ucanto/principal' +import { equals } from 'uint8arrays' +import { sha256 } from 'multiformats/hashes/sha2' +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import { base64pad } from 'multiformats/bases/base64' + +import { provisionProvider } from '../helpers/utils.js' +import { createServer, connect } from '../../src/lib.js' +import { alice, bob, createSpace, registerSpace } from '../util.js' +import { BlobItemSizeExceededName } from '../../src/blob/lib.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'blob/add schedules allocation and returns effects for allocation and accept': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // invoke `blob/add` + const invocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [proof], + }) + const blobAdd = await invocation.execute(connection) + if (!blobAdd.out.ok) { + console.log('out error') + throw new Error('invocation failed', { cause: blobAdd }) + } + + assert.ok(blobAdd.out.ok.claim) + assert.ok(blobAdd.fx.fork.length) + assert.ok(blobAdd.fx.join) + assert.ok(blobAdd.out.ok.claim['await/ok'].equals(blobAdd.fx.join)) + + // validate scheduled task ran + // await deferredSchedule.promise + // assert.equal(scheduledTasks.length, 1) + // const [blobAllocateInvocation] = scheduledTasks + // assert.equal(blobAllocateInvocation.can, BlobCapabilities.allocate.can) + // assert.equal(blobAllocateInvocation.nb.space, spaceDid) + // assert.equal(blobAllocateInvocation.nb.blob.size, size) + // assert.ok(equals(blobAllocateInvocation.nb.blob.content, content)) + }, + 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // invoke `blob/add` + const invocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size: Number.MAX_SAFE_INTEGER + }, + }, + proofs: [proof], + }) + const blobAdd = await invocation.execute(connection) + if (!blobAdd.out.error) { + throw new Error('invocation should have failed') + } + assert.ok(blobAdd.out.error, 'invocation should have failed') + assert.equal(blobAdd.out.error.name, BlobItemSizeExceededName) + }, + 'skip blob/add fails when allocate task cannot be scheduled': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // invoke `blob/add` + const invocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [proof], + }) + const blobAdd = await invocation.execute(connection) + if (!blobAdd.out.error) { + throw new Error('invocation should have failed') + } + assert.ok(blobAdd.out.error, 'invocation should have failed') + assert.ok(blobAdd.out.error.message.includes(BlobCapabilities.allocate.can)) + assert.equal(blobAdd.out.error.name, 'Error') + }, + 'blob/allocate allocates to space and returns presigned url': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const digest = multihash.digest + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + + // Validate response + assert.equal(blobAllocate.out.ok.size, size) + assert.ok(blobAllocate.out.ok.address) + assert.ok(blobAllocate.out.ok.address?.headers) + assert.ok(blobAllocate.out.ok.address?.url) + assert.equal(blobAllocate.out.ok.address?.headers?.['content-length'], String(size)) + assert.deepEqual( + blobAllocate.out.ok.address?.headers?.['x-amz-checksum-sha256'], + base64pad.baseEncode(digest) + ) + + const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const signedHeaders = url.searchParams.get('X-Amz-SignedHeaders') + + assert.equal( + signedHeaders, + 'content-length;host;x-amz-checksum-sha256', + 'content-length and checksum must be part of the signature' + ) + + // Validate allocation state + const spaceAllocations = await context.allocationStorage.list(spaceDid) + assert.ok(spaceAllocations.ok) + assert.equal(spaceAllocations.ok?.size, 1) + const allocatedEntry = spaceAllocations.ok?.results[0] + if (!allocatedEntry) { + throw new Error('Expected presigned allocatedEntry in response') + } + assert.ok(equals(allocatedEntry.blob.content, content)) + assert.equal(allocatedEntry.blob.size, size) + + // Validate presigned url usage + const goodPut = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: blobAllocate.out.ok.address?.headers, + }) + + assert.equal(goodPut.status, 200, await goodPut.text()) + }, + 'blob/allocate does not allocate more space to already allocated content': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + + // second blob allocate invocation + const secondBlobAllocate = await serviceBlobAllocate.execute(connection) + if (!secondBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: secondBlobAllocate }) + } + + // Validate response + assert.equal(secondBlobAllocate.out.ok.size, 0) + assert.ok(!!blobAllocate.out.ok.address) + }, + 'blob/allocate can allocate to different space after write to one space': async (assert, context) => { + const { proof: aliceProof, spaceDid: aliceSpaceDid } = await registerSpace(alice, context) + const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( + bob, + context, + 'bob' + ) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocations + const aliceBlobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [aliceProof], + }) + const bobBlobAddInvocation = BlobCapabilities.add.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [bobProof], + }) + + // invoke `service/blob/allocate` capabilities on alice space + const aliceServiceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: { + content, + size + }, + cause: (await aliceBlobAddInvocation.delegate()).cid, + space: aliceSpaceDid, + }, + proofs: [aliceProof], + }) + const aliceBlobAllocate = await aliceServiceBlobAllocate.execute(connection) + if (!aliceBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: aliceBlobAllocate }) + } + // there is address to write + assert.ok(aliceBlobAllocate.out.ok.address) + assert.equal(aliceBlobAllocate.out.ok.size, size) + + // write to presigned url + const url = aliceBlobAllocate.out.ok.address?.url && new URL(aliceBlobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const goodPut = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: aliceBlobAllocate.out.ok.address?.headers, + }) + + assert.equal(goodPut.status, 200, await goodPut.text()) + + // invoke `service/blob/allocate` capabilities on bob space + const bobServiceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: { + content, + size + }, + cause: (await bobBlobAddInvocation.delegate()).cid, + space: bobSpaceDid, + }, + proofs: [bobProof], + }) + const bobBlobAllocate = await bobServiceBlobAllocate.execute(connection) + if (!bobBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: bobBlobAllocate }) + } + // there is no address to write + assert.ok(!bobBlobAllocate.out.ok.address) + assert.equal(bobBlobAllocate.out.ok.size, size) + + // Validate allocation state + const aliceSpaceAllocations = await context.allocationStorage.list(aliceSpaceDid) + assert.ok(aliceSpaceAllocations.ok) + assert.equal(aliceSpaceAllocations.ok?.size, 1) + + const bobSpaceAllocations = await context.allocationStorage.list(bobSpaceDid) + assert.ok(bobSpaceAllocations.ok) + assert.equal(bobSpaceAllocations.ok?.size, 1) + }, + 'blob/allocate creates presigned url that can only PUT a payload with right length': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + // there is address to write + assert.ok(blobAllocate.out.ok.address) + assert.equal(blobAllocate.out.ok.size, size) + + // write to presigned url + const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const contentLengthFailSignature = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: longer, + headers: { + ...blobAllocate.out.ok.address?.headers, + 'content-length': longer.byteLength.toString(10), + }, + }) + + assert.equal( + contentLengthFailSignature.status >= 400, + true, + 'should fail to upload as content-length differs from that used to sign the url' + ) + }, + 'blob/allocate creates presigned url that can only PUT a payload with exact bytes': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const other = new Uint8Array([10, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + // there is address to write + assert.ok(blobAllocate.out.ok.address) + assert.equal(blobAllocate.out.ok.size, size) + + // write to presigned url + const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const failChecksum = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: other, + headers: blobAllocate.out.ok.address?.headers, + }) + + assert.equal( + failChecksum.status, + 400, + 'should fail to upload any other data.' + ) + }, + 'blob/allocate disallowed if invocation fails access verification': async (assert, context) => { + const { proof, space, spaceDid } = await createSpace(alice) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + }, + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + assert.ok(blobAllocate.out.error) + assert.equal(blobAllocate.out.error?.message.includes('no storage'), true) + + // Register space and retry + const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) + const providerAdd = await provisionProvider({ + service: /** @type {API.Signer>} */ (context.signer), + agent: alice, + space, + account, + connection, + }) + assert.ok(providerAdd.out.ok) + + const retryBlobAllocate = await serviceBlobAllocate.execute(connection) + assert.equal(retryBlobAllocate.out.error, undefined) + }, + // TODO: Blob accept + // TODO: list + // TODO: remove +} diff --git a/packages/upload-api/test/handlers/blob.spec.js b/packages/upload-api/test/handlers/blob.spec.js new file mode 100644 index 000000000..c8bd740b7 --- /dev/null +++ b/packages/upload-api/test/handlers/blob.spec.js @@ -0,0 +1,4 @@ +import { test } from '../test.js' +import * as Blob from './blob.js' + +test({ 'blob/*': Blob.test }) diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index 0c126d122..6176b318b 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -5,6 +5,8 @@ import { getStoreImplementations, getQueueImplementations, } from '@web3-storage/filecoin-api/test/context/service' +import { AllocationStorage } from '../storage/allocation-storage.js' +import { BlobStorage } from '../storage/blob-storage.js' import { CarStoreBucket } from '../storage/car-store-bucket.js' import { StoreTable } from '../storage/store-table.js' import { UploadTable } from '../storage/upload-table.js' @@ -36,7 +38,9 @@ export const createContext = async ( ) => { const requirePaymentPlan = options.requirePaymentPlan const storeTable = new StoreTable() + const allocationStorage = new AllocationStorage() const uploadTable = new UploadTable() + const blobStorage = await BlobStorage.activate(options) const carStoreBucket = await CarStoreBucket.activate(options) const dudewhereBucket = new DudewhereBucket() const revocationsStorage = new RevocationsStorage() @@ -44,6 +48,8 @@ export const createContext = async ( const usageStorage = new UsageStorage(storeTable) const provisionsStorage = new ProvisionsStorage(options.providers) const subscriptionsStorage = new SubscriptionsStorage(provisionsStorage) + const delegationsStorage = new DelegationsStorage() + const rateLimitsStorage = new RateLimitsStorage() const signer = await Signer.generate() const aggregatorSigner = await Signer.generate() const dealTrackerSigner = await Signer.generate() @@ -74,8 +80,8 @@ export const createContext = async ( url: new URL('http://localhost:8787'), provisionsStorage, subscriptionsStorage, - delegationsStorage: new DelegationsStorage(), - rateLimitsStorage: new RateLimitsStorage(), + delegationsStorage, + rateLimitsStorage, plansStorage, usageStorage, revocationsStorage, @@ -90,8 +96,10 @@ export const createContext = async ( }, maxUploadSize: 5_000_000_000, storeTable, + allocationStorage, uploadTable, carStoreBucket, + blobStorage, dudewhereBucket, filecoinSubmitQueue, pieceOfferQueue, @@ -107,6 +115,7 @@ export const createContext = async ( audience: dealTrackerSigner, }, }, + getServiceConnection: () => connection, ...createRevocationChecker({ revocationsStorage }), } @@ -132,7 +141,11 @@ export const createContext = async ( export const cleanupContext = async (context) => { /** @type {CarStoreBucket & { deactivate: () => Promise }}} */ // @ts-ignore type misses S3 bucket properties like accessKey - const store = context.carStoreBucket + const carStoreBucket = context.carStoreBucket + await carStoreBucket.deactivate() - await store.deactivate() + /** @type {BlobStorage & { deactivate: () => Promise }}} */ + // @ts-ignore type misses S3 bucket properties like accessKey + const blobStorage = context.blobStorage + await blobStorage.deactivate() } diff --git a/packages/upload-api/test/lib.js b/packages/upload-api/test/lib.js index 1d830e8cc..15425b86c 100644 --- a/packages/upload-api/test/lib.js +++ b/packages/upload-api/test/lib.js @@ -7,6 +7,7 @@ import * as RateLimitAdd from './handlers/rate-limit/add.js' import * as RateLimitList from './handlers/rate-limit/list.js' import * as RateLimitRemove from './handlers/rate-limit/remove.js' import * as Store from './handlers/store.js' +import * as Blob from './handlers/blob.js' import * as Subscription from './handlers/subscription.js' import * as Upload from './handlers/upload.js' import * as Plan from './handlers/plan.js' @@ -23,6 +24,7 @@ export * from './util.js' export const test = { ...Store.test, + ...Blob.test, ...Upload.test, } @@ -44,6 +46,7 @@ export const handlerTests = { ...RateLimitList, ...RateLimitRemove, ...Store.test, + ...Blob.test, ...Subscription.test, ...Upload.test, ...Plan.test, diff --git a/packages/upload-api/test/storage/allocation-storage.js b/packages/upload-api/test/storage/allocation-storage.js new file mode 100644 index 000000000..7d30d0a46 --- /dev/null +++ b/packages/upload-api/test/storage/allocation-storage.js @@ -0,0 +1,109 @@ +import * as Types from '../../src/types.js' +import { equals } from 'uint8arrays/equals' + +/** + * @implements {Types.AllocationStorage} + */ +export class AllocationStorage { + constructor() { + /** @type {(Types.BlobAddInput & Types.BlobListItem)[]} */ + this.items = [] + } + + /** + * @param {Types.BlobAddInput} input + * @returns {ReturnType} + */ + async insert({ space, invocation, ...output }) { + if ( + this.items.some( + (i) => i.space === space && equals(i.blob.content, output.blob.content) + ) + ) { + return { + error: { name: 'RecordKeyConflict', message: 'record key conflict' }, + } + } + this.items.unshift({ + space, + invocation, + ...output, + insertedAt: new Date().toISOString(), + }) + return { ok: output } + } + + /** + * @param {Types.DID} space + * @param {Uint8Array} blobMultihash + * @returns {ReturnType} + */ + async exists(space, blobMultihash) { + const item = this.items.find( + (i) => i.space === space && equals(i.blob.content, blobMultihash) + ) + return { ok: !!item } + } + + /** + * @param {Types.DID} space + * @param {Uint8Array} blobMultihash + * @returns {ReturnType} + */ + async remove(space, blobMultihash) { + const item = this.items.find( + (i) => i.space === space && equals(i.blob.content, blobMultihash) + ) + if (!item) { + return { error: { name: 'RecordNotFound', message: 'record not found' } } + } + this.items = this.items.filter((i) => i !== item) + return { + ok: { + size: item.blob.size, + }, + } + } + + /** + * @param {Types.DID} space + * @param {Types.ListOptions} options + * @returns {ReturnType} + */ + async list( + space, + { cursor = '0', pre = false, size = this.items.length } = {} + ) { + const offset = parseInt(cursor, 10) + const items = pre ? this.items.slice(0, offset) : this.items.slice(offset) + + const matches = [...items.entries()] + .filter(([n, item]) => item.space === space) + .slice(0, size) + + if (matches.length === 0) { + return { ok: { size: 0, results: [] } } + } + + const first = matches[0] + const last = matches[matches.length - 1] + + const start = first[0] || 0 + const end = last[0] || 0 + const values = matches.map(([_, item]) => item) + + const [before, after, results] = pre + ? [`${start}`, `${end + 1}`, values] + : [`${start + offset}`, `${end + 1 + offset}`, values] + + return { + ok: { + size: values.length, + before, + after, + cursor: after, + results, + }, + } + } +} diff --git a/packages/upload-api/test/storage/blob-storage.js b/packages/upload-api/test/storage/blob-storage.js new file mode 100644 index 000000000..807d029d7 --- /dev/null +++ b/packages/upload-api/test/storage/blob-storage.js @@ -0,0 +1,169 @@ +import * as Types from '../../src/types.js' + +import { base64pad } from 'multiformats/bases/base64' +import { decode as digestDecode } from 'multiformats/hashes/digest' +import { SigV4 } from '@web3-storage/sigv4' +import { base58btc } from 'multiformats/bases/base58' +import { sha256 } from 'multiformats/hashes/sha2' + +/** + * @implements {Types.BlobStorage} + */ +export class BlobStorage { + /** + * @param {Types.CarStoreBucketOptions & {http?: import('http')}} options + */ + static async activate({ http, ...options } = {}) { + const content = new Map() + if (http) { + const server = http.createServer(async (request, response) => { + if (request.method === 'PUT') { + const buffer = new Uint8Array( + parseInt(request.headers['content-length'] || '0') + ) + let offset = 0 + for await (const chunk of request) { + buffer.set(chunk, offset) + offset += chunk.length + } + const hash = await sha256.digest(buffer) + const checksum = base64pad.baseEncode(hash.digest) + + if (checksum !== request.headers['x-amz-checksum-sha256']) { + response.writeHead(400, `checksum mismatch`) + } else { + const { pathname } = new URL(request.url || '/', url) + content.set(pathname, buffer) + response.writeHead(200) + } + } else { + response.writeHead(405) + } + + response.end() + // otherwise it keep connection lingering + response.destroy() + }) + await new Promise((resolve) => server.listen(resolve)) + + // @ts-ignore - this is actually what it returns on http + const port = server.address().port + const url = new URL(`http://localhost:${port}`) + + return new BlobStorage({ + ...options, + content, + url, + server, + }) + } else { + return new BlobStorage({ + ...options, + content, + url: new URL(`http://localhost:8989`), + }) + } + } + + /** + * @returns {Promise} + */ + async deactivate() { + const { server } = this + if (server) { + await new Promise((resolve, reject) => { + // does not exist in node 16 + if (typeof server.closeAllConnections === 'function') { + server.closeAllConnections() + } + + server.close((error) => { + if (error) { + reject(error) + } else { + resolve(undefined) + } + }) + }) + } + } + + /** + * @param {Types.CarStoreBucketOptions & { server?: import('http').Server, url: URL, content: Map }} options + */ + constructor({ + content, + url, + server, + accessKeyId = 'id', + secretAccessKey = 'secret', + bucket = 'my-bucket', + region = 'eu-central-1', + expires, + }) { + this.server = server + this.baseURL = url + this.accessKeyId = accessKeyId + this.secretAccessKey = secretAccessKey + this.bucket = bucket + this.region = region + this.expires = expires + this.content = content + } + + /** + * @param {Uint8Array} multihash + */ + async has(multihash) { + const encodedMultihash = base58btc.encode(multihash) + return { + ok: this.content.has( + `/${this.bucket}/${encodedMultihash}/${encodedMultihash}.blob` + ), + } + } + + /** + * @param {Uint8Array} multihash + * @param {number} size + */ + async createUploadUrl(multihash, size) { + const { bucket, expires, accessKeyId, secretAccessKey, region, baseURL } = + this + const encodedMultihash = base58btc.encode(multihash) + const multihashDigest = digestDecode(multihash) + // sigv4 + const sig = new SigV4({ + accessKeyId, + secretAccessKey, + region, + }) + + const checksum = base64pad.baseEncode(multihashDigest.digest) + const { pathname, search, hash } = sig.sign({ + key: `${encodedMultihash}/${encodedMultihash}.blob`, + checksum, + bucket, + expires, + }) + + const url = new URL(baseURL) + url.search = search + url.pathname = `/${bucket}${pathname}` + url.hash = hash + url.searchParams.set( + 'X-Amz-SignedHeaders', + ['content-length', 'host', 'x-amz-checksum-sha256'].join(';') + ) + + return { + ok: { + url, + headers: { + 'x-amz-checksum-sha256': checksum, + 'content-length': String(size), + }, + }, + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4285e1c07..5fdd31cc8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -169,6 +169,9 @@ importers: '@web3-storage/data-segment': specifier: ^3.2.0 version: 3.2.0 + uint8arrays: + specifier: ^5.0.3 + version: 5.0.3 devDependencies: '@types/assert': specifier: ^1.5.6 @@ -412,6 +415,9 @@ importers: p-retry: specifier: ^5.1.2 version: 5.1.2 + uint8arrays: + specifier: ^5.0.3 + version: 5.0.3 devDependencies: '@ipld/car': specifier: ^5.1.1 @@ -422,6 +428,9 @@ importers: '@types/mocha': specifier: ^10.0.1 version: 10.0.4 + '@types/sinon': + specifier: ^17.0.3 + version: 17.0.3 '@ucanto/core': specifier: ^10.0.1 version: 10.0.1 @@ -4088,6 +4097,12 @@ packages: '@types/sinonjs__fake-timers': 8.1.5 dev: true + /@types/sinon@17.0.3: + resolution: {integrity: sha512-j3uovdn8ewky9kRBG19bOwaZbexJu/XjtkHyjvUgt4xfPFz18dcORIMqnYh66Fx3Powhcr85NT5+er3+oViapw==} + dependencies: + '@types/sinonjs__fake-timers': 8.1.5 + dev: true + /@types/sinonjs__fake-timers@8.1.5: resolution: {integrity: sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==} dev: true From dece9c49b60c25885f7896ab4d07e4f173de96a2 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 3 Apr 2024 11:03:40 +0200 Subject: [PATCH 02/27] feat: http put --- packages/capabilities/src/blob.js | 41 + packages/capabilities/src/index.js | 2 + packages/capabilities/src/types.ts | 12 + packages/capabilities/src/ucan.js | 53 ++ packages/capabilities/src/utils.js | 2 +- packages/upload-api/src/blob/accept.js | 2 +- packages/upload-api/src/blob/add.js | 113 ++- packages/upload-api/src/blob/allocate.js | 6 +- packages/upload-api/src/blob/list.js | 2 +- packages/upload-api/src/blob/remove.js | 2 +- packages/upload-api/src/errors.js | 25 + packages/upload-api/src/service.js | 2 +- packages/upload-api/src/types.ts | 84 +- packages/upload-api/src/types/blob.ts | 75 ++ packages/upload-api/src/types/service.ts | 4 + packages/upload-api/src/types/storage.ts | 34 + packages/upload-api/src/ucan.js | 2 + packages/upload-api/src/ucan/conclude.js | 18 + packages/upload-api/test/handlers/blob.js | 861 +++++++++--------- packages/upload-api/test/helpers/context.js | 66 +- ...tion-storage.js => allocations-storage.js} | 12 +- .../{blob-storage.js => blobs-storage.js} | 8 +- packages/upload-api/test/storage/index.js | 58 ++ .../test/storage/receipts-storage.js | 64 ++ .../upload-api/test/storage/tasks-storage.js | 64 ++ 25 files changed, 1022 insertions(+), 590 deletions(-) create mode 100644 packages/upload-api/src/errors.js create mode 100644 packages/upload-api/src/types/blob.ts create mode 100644 packages/upload-api/src/types/service.ts create mode 100644 packages/upload-api/src/types/storage.ts create mode 100644 packages/upload-api/src/ucan/conclude.js rename packages/upload-api/test/storage/{allocation-storage.js => allocations-storage.js} (88%) rename packages/upload-api/test/storage/{blob-storage.js => blobs-storage.js} (97%) create mode 100644 packages/upload-api/test/storage/index.js create mode 100644 packages/upload-api/test/storage/receipts-storage.js create mode 100644 packages/upload-api/test/storage/tasks-storage.js diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 3a96a7e3e..331dd893b 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -188,6 +188,47 @@ export const allocate = capability({ }, }) +/** + * `http/put` capability invocation MAY be performed by any agent on behalf of the subject. + * The `blob/add` provider MUST add `/http/put` effect and capture private key of the + * `subject` in the `meta` field so that any agent could perform it. + */ +export const put = capability({ + can: 'http/put', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * A multihash digest of the blob payload bytes, uniquely identifying blob. + */ + content: Schema.bytes(), + /** + * Blob to accept. + */ + address: Schema.struct({ + /** + * HTTP(S) location that can receive blob content via HTTP PUT request. + */ + url: Schema.string(), + /** + * HTTP headers. + */ + headers: Schema.unknown(), + }).optional(), + }), + derives: (claim, from) => { + return ( + and(equalContent(claim, from)) || + and(equal(claim.nb.address?.url, from.nb.address, 'url')) || + and(equal(claim.nb.address?.headers, from.nb.address, 'headers')) || + ok({}) + ) + }, +}) + /** * `blob/accept` capability invocation should either succeed when content is * delivered on allocated address or fail if no content is allocation expires diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 8c423bfe1..fc7e4bc7c 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -64,6 +64,7 @@ export const abilitiesAsStrings = [ Access.access.can, Access.authorize.can, UCAN.attest.can, + UCAN.conclude.can, Customer.get.can, Consumer.has.can, Consumer.get.can, @@ -92,6 +93,7 @@ export const abilitiesAsStrings = [ Blob.remove.can, Blob.list.can, Blob.serviceBlob.can, + Blob.put.can, Blob.allocate.can, Blob.accept.can, ] diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 9944af005..64c99825c 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -446,6 +446,7 @@ export type BlobAdd = InferInvokedCapability export type BlobRemove = InferInvokedCapability export type BlobList = InferInvokedCapability export type ServiceBlob = InferInvokedCapability +export type BlobPut = InferInvokedCapability export type BlobAllocate = InferInvokedCapability export type BlobAccept = InferInvokedCapability @@ -605,6 +606,7 @@ export interface UploadListSuccess extends ListResponse {} export type UCANRevoke = InferInvokedCapability export type UCANAttest = InferInvokedCapability +export type UCANConclude = InferInvokedCapability export interface Timestamp { /** @@ -615,6 +617,8 @@ export interface Timestamp { export type UCANRevokeSuccess = Timestamp +export type UCANConcludeSuccess = Timestamp + /** * Error is raised when `UCAN` being revoked is not supplied or it's proof chain * leading to supplied `scope` is not supplied. @@ -653,6 +657,12 @@ export type UCANRevokeFailure = | UnauthorizedRevocation | RevocationsStoreFailure +export interface InvocationNotFound extends Ucanto.Failure { + name: 'InvocationNotFound' +} + +export type UCANConcludeFailure = InvocationNotFound | Ucanto.Failure + // Admin export type Admin = InferInvokedCapability export type AdminUploadInspect = InferInvokedCapability< @@ -761,6 +771,7 @@ export type ServiceAbilityArray = [ Access['can'], AccessAuthorize['can'], UCANAttest['can'], + UCANConclude['can'], CustomerGet['can'], ConsumerHas['can'], ConsumerGet['can'], @@ -789,6 +800,7 @@ export type ServiceAbilityArray = [ BlobRemove['can'], BlobList['can'], ServiceBlob['can'], + BlobPut['can'], BlobAllocate['can'], BlobAccept['can'] ] diff --git a/packages/capabilities/src/ucan.js b/packages/capabilities/src/ucan.js index fe38b757e..83ce637fa 100644 --- a/packages/capabilities/src/ucan.js +++ b/packages/capabilities/src/ucan.js @@ -74,6 +74,59 @@ export const revoke = capability({ ), }) +/** + * `ucan/conclude` capability represents a receipt using a special UCAN capability. + * + * The UCAN invocation specification defines receipt record, that is cryptographically + * signed description of the invocation output and requested effects. Receipt + * structure is very similar to UCAN except it has no notion of expiry nor it is + * possible to delegate ability to issue receipt to another principal. + */ +export const conclude = capability({ + can: 'ucan/conclude', + /** + * DID of the principal representing the Conclusion Authority. + * MUST be the DID of the audience of the ran invocation. + */ + with: Schema.did(), + // TODO: Should this just have bytes? + nb: Schema.struct({ + /** + * A link to the UCAN invocation that this receipt is for. + */ + ran: UCANLink, + /** + * The value output of the invocation in Result format. + */ + out: Schema.unknown(), + /** + * Tasks that the invocation would like to enqueue. + */ + next: Schema.array(UCANLink), + /** + * Additional data about the receipt + */ + meta: Schema.unknown(), + /** + * The UTC Unix timestamp at which the Receipt was issued + */ + time: Schema.integer(), + }), + derives: (claim, from) => + // With field MUST be the same + and(equalWith(claim, from)) ?? + // invocation MUST be the same + and(checkLink(claim.nb.ran, from.nb.ran, 'nb.ran')) ?? + // value output MUST be the same + and(equal(claim.nb.out, from.nb.out, 'nb.out')) ?? + // tasks to enqueue MUST be the same + and(equal(claim.nb.next, from.nb.next, 'nb.next')) ?? + // additional data MUST be the same + and(equal(claim.nb.meta, from.nb.meta, 'nb.meta')) ?? + // the receipt issue time MUST be the same + equal(claim.nb.time, from.nb.time, 'nb.time'), +}) + /** * Issued by trusted authority (usually the one handling invocation) that attest * that specific UCAN delegation has been considered authentic. diff --git a/packages/capabilities/src/utils.js b/packages/capabilities/src/utils.js index 96c63c352..0db700c48 100644 --- a/packages/capabilities/src/utils.js +++ b/packages/capabilities/src/utils.js @@ -122,7 +122,7 @@ export const equalBlob = (claimed, delegated) => { } /** - * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"blob/allocate"|"blob/accept", Types.URI<'did:'>, {content: Uint8Array}>} T + * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"blob/allocate"|"blob/accept"|"http/put", Types.URI<'did:'>, {content: Uint8Array}>} T * @param {T} claimed * @param {T} delegated * @returns {Types.Result<{}, Types.Failure>} diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index 6c86e7e1d..07113aaa3 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -11,7 +11,7 @@ export function blobAcceptProvider(context) { return Server.provide(Blob.accept, async ({ capability }) => { const { blob } = capability.nb // If blob is not stored, we must fail - const hasBlob = await context.blobStorage.has(blob.content) + const hasBlob = await context.blobsStorage.has(blob.content) if (hasBlob.error) { return { error: new BlobItemNotFound(), diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 3c9825a36..f46e5e06d 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -1,4 +1,5 @@ import * as Server from '@ucanto/server' +import { ed25519 } from '@ucanto/principal' import * as Blob from '@web3-storage/capabilities/blob' import * as API from '../types.js' @@ -12,7 +13,13 @@ export function blobAddProvider(context) { return Server.provideAdvanced({ capability: Blob.add, handler: async ({ capability, invocation }) => { - const { id, allocationStorage, maxUploadSize, getServiceConnection } = context + const { + id, + allocationsStorage, + maxUploadSize, + getServiceConnection, + tasksStorage, + } = context const { blob } = capability.nb const space = /** @type {import('@ucanto/interface').DIDKey} */ ( Server.DID.parse(capability.with).did() @@ -20,48 +27,88 @@ export function blobAddProvider(context) { if (blob.size > maxUploadSize) { return { - error: new BlobItemSizeExceeded(maxUploadSize) + error: new BlobItemSizeExceeded(maxUploadSize), } } - // Create effects for receipt - // TODO: needs HTTP/PUT receipt - const blobAllocate = Blob.allocate - .invoke({ - issuer: id, - audience: id, - with: id.did(), - nb: { - blob, - cause: invocation.link(), - space, - }, - expiration: Infinity - }) - const blobAccept = Blob.accept - .invoke({ - issuer: id, - audience: id, - with: id.toDIDKey(), - nb: { - blob, - exp: Number.MAX_SAFE_INTEGER, - }, - expiration: Infinity, + const putSubject = await ed25519.derive(blob.content.slice(0, 32)) + const facts = Object.entries(putSubject.toArchive().keys).map( + ([key, value]) => ({ + did: key, + bytes: value, }) - const [allocatefx, acceptfx] = await Promise.all([ + ) + + // Create effects for receipt + const blobAllocate = Blob.allocate.invoke({ + issuer: id, + audience: id, + with: id.did(), + nb: { + blob, + cause: invocation.link(), + space, + }, + expiration: Infinity, + }) + const blobPut = Blob.put.invoke({ + issuer: putSubject, + audience: putSubject, + with: putSubject.toDIDKey(), + nb: { + content: blob.content, + }, + facts, + expiration: Infinity, + }) + const blobAccept = Blob.accept.invoke({ + issuer: id, + audience: id, + with: id.toDIDKey(), + nb: { + blob, + exp: Number.MAX_SAFE_INTEGER, + }, + expiration: Infinity, + }) + const [allocatefx, putfx, acceptfx] = await Promise.all([ + // 1. System attempts to allocate memory in user space for the blob. blobAllocate.delegate(), + // 2. System requests user agent (or anyone really) to upload the content + // corresponding to the blob + // via HTTP PUT to given location. + blobPut.delegate(), + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. blobAccept.delegate(), ]) + // store `http/put` invocation + // TODO: store implementation + // const archiveDelegationRes = await putfx.archive() + // if (archiveDelegationRes.error) { + // return { + // error: archiveDelegationRes.error + // } + // } + const invocationPutRes = await tasksStorage.put(putfx) + if (invocationPutRes.error) { + return { + error: invocationPutRes.error, + } + } + // Schedule allocation if not allocated - const allocated = await allocationStorage.exists(space, blob.content) - if (!allocated.ok) { + const allocatedExistsRes = await allocationsStorage.exists( + space, + blob.content + ) + if (!allocatedExistsRes.ok) { // Execute allocate invocation const allocateRes = await blobAllocate.execute(getServiceConnection()) if (allocateRes.out.error) { return { - error: allocateRes.out.error + error: allocateRes.out.error, } } } @@ -72,7 +119,11 @@ export function blobAddProvider(context) { 'await/ok': acceptfx.link(), }, }) - return result.fork(allocatefx.link()).join(acceptfx.link()) + // TODO: not pass links, but delegation + return result + .fork(allocatefx.link()) + .fork(putfx.link()) + .join(acceptfx.link()) }, }) } diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index 0244f80e3..0693d3d3a 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -43,14 +43,14 @@ export function blobAllocateProvider(context) { } // If blob is stored, we can just allocate it to the space - const hasBlob = await context.blobStorage.has(blob.content) + const hasBlob = await context.blobsStorage.has(blob.content) if (hasBlob.error) { return { error: new BlobItemNotFound(space), } } // Get presigned URL for the write target - const createUploadUrl = await context.blobStorage.createUploadUrl( + const createUploadUrl = await context.blobsStorage.createUploadUrl( blob.content, blob.size ) @@ -61,7 +61,7 @@ export function blobAllocateProvider(context) { } // Allocate in space, ignoring if already allocated - const allocationInsert = await context.allocationStorage.insert({ + const allocationInsert = await context.allocationsStorage.insert({ space, blob, invocation: cause, diff --git a/packages/upload-api/src/blob/list.js b/packages/upload-api/src/blob/list.js index 6694804fd..c6c34bdb7 100644 --- a/packages/upload-api/src/blob/list.js +++ b/packages/upload-api/src/blob/list.js @@ -10,6 +10,6 @@ export function blobListProvider(context) { return Server.provide(Blob.list, async ({ capability }) => { const { cursor, size, pre } = capability.nb const space = Server.DID.parse(capability.with).did() - return await context.allocationStorage.list(space, { size, cursor, pre }) + return await context.allocationsStorage.list(space, { size, cursor, pre }) }) } diff --git a/packages/upload-api/src/blob/remove.js b/packages/upload-api/src/blob/remove.js index 546f4935f..fb2e8c2d8 100644 --- a/packages/upload-api/src/blob/remove.js +++ b/packages/upload-api/src/blob/remove.js @@ -12,7 +12,7 @@ export function blobRemoveProvider(context) { const { content } = capability.nb const space = Server.DID.parse(capability.with).did() - const res = await context.allocationStorage.remove(space, content) + const res = await context.allocationsStorage.remove(space, content) if (res.error && res.error.name === 'RecordNotFound') { return Server.error(new BlobItemNotFound(space)) } diff --git a/packages/upload-api/src/errors.js b/packages/upload-api/src/errors.js new file mode 100644 index 000000000..13620c3a1 --- /dev/null +++ b/packages/upload-api/src/errors.js @@ -0,0 +1,25 @@ +import * as Server from '@ucanto/server' + +export const StoreOperationErrorName = /** @type {const} */ ( + 'StoreOperationFailed' +) +export class StoreOperationFailed extends Server.Failure { + get reason() { + return this.message + } + + get name() { + return StoreOperationErrorName + } +} + +export const RecordNotFoundErrorName = /** @type {const} */ ('RecordNotFound') +export class RecordNotFound extends Server.Failure { + get reason() { + return this.message + } + + get name() { + return RecordNotFoundErrorName + } +} diff --git a/packages/upload-api/src/service.js b/packages/upload-api/src/service.js index bb9c503bc..e650e13d0 100644 --- a/packages/upload-api/src/service.js +++ b/packages/upload-api/src/service.js @@ -10,6 +10,6 @@ export function createService(context) { blob: { allocate: blobAllocateProvider(context), accept: blobAcceptProvider(context), - } + }, } } diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 073d1a972..3c932354f 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -54,7 +54,6 @@ export interface DebugEmail extends Email { } import { - BlobMultihash, BlobAdd, BlobAddSuccess, BlobAddFailure, @@ -62,7 +61,6 @@ import { BlobRemoveSuccess, BlobRemoveFailure, BlobList, - BlobListItem, BlobListSuccess, BlobListFailure, BlobAllocate, @@ -178,6 +176,15 @@ import { SubscriptionsStorage } from './types/subscriptions.js' export type { SubscriptionsStorage } import { UsageStorage } from './types/usage.js' export type { UsageStorage } +import { ReceiptsStorage } from './types/service.js' +export type { ReceiptsStorage } +import { + AllocationsStorage, + BlobsStorage, + TasksStorage, + BlobAddInput, +} from './types/blob.js' +export type { AllocationsStorage, BlobsStorage, TasksStorage, BlobAddInput } export interface Service extends StorefrontService, W3sService { blob: { @@ -314,8 +321,9 @@ export type BlobServiceContext = SpaceServiceContext & { */ id: Signer maxUploadSize: number - allocationStorage: AllocationStorage - blobStorage: BlobStorage + allocationsStorage: AllocationsStorage + blobsStorage: BlobsStorage + tasksStorage: TasksStorage getServiceConnection: () => ConnectionView } @@ -324,8 +332,8 @@ export type W3ServiceContext = SpaceServiceContext & { * Service signer */ id: Signer - allocationStorage: AllocationStorage - blobStorage: BlobStorage + allocationsStorage: AllocationsStorage + blobsStorage: BlobsStorage } export type StoreServiceContext = SpaceServiceContext & { @@ -336,7 +344,8 @@ export type StoreServiceContext = SpaceServiceContext & { export type UploadServiceContext = ConsumerServiceContext & SpaceServiceContext & - RevocationServiceContext & { + RevocationServiceContext & + ConcludeServiceContext & { signer: EdSigner.Signer uploadTable: UploadTable dudewhereBucket: DudewhereBucket @@ -399,6 +408,13 @@ export interface RevocationServiceContext { revocationsStorage: RevocationsStorage } +export interface ConcludeServiceContext { + /** + * Stores receipts for tasks. + */ + receiptsStorage: ReceiptsStorage +} + export interface PlanServiceContext { plansStorage: PlansStorage } @@ -417,6 +433,7 @@ export interface ServiceContext SpaceServiceContext, StoreServiceContext, BlobServiceContext, + ConcludeServiceContext, SubscriptionServiceContext, RateLimitServiceContext, RevocationServiceContext, @@ -451,25 +468,6 @@ export interface ErrorReporter { catch: (error: HandlerExecutionError) => void } -export interface BlobStorage { - has: (content: BlobMultihash) => Promise> - createUploadUrl: ( - content: BlobMultihash, - size: number - ) => Promise< - Result< - { - url: URL - headers: { - 'x-amz-checksum-sha256': string - 'content-length': string - } & Record - }, - Failure - > - > -} - export interface CarStoreBucket { has: (link: UnknownLink) => Promise createUploadUrl: ( @@ -516,26 +514,6 @@ export interface RecordKeyConflict extends Failure { name: 'RecordKeyConflict' } -export interface AllocationStorage { - exists: ( - space: DID, - blobMultihash: BlobMultihash - ) => Promise> - /** Inserts an item in the table if it does not already exist. */ - insert: ( - item: BlobAddInput - ) => Promise> - /** Removes an item from the table but fails if the item does not exist. */ - remove: ( - space: DID, - blobMultihash: BlobMultihash - ) => Promise> - list: ( - space: DID, - options?: ListOptions - ) => Promise, Failure>> -} - export interface StoreTable { inspect: (link: UnknownLink) => Promise> exists: (space: DID, link: UnknownLink) => Promise> @@ -604,20 +582,6 @@ export type AdminUploadInspectResult = Result< AdminUploadInspectFailure > -export interface Blob { - content: BlobMultihash - size: number -} - -export interface BlobAddInput { - space: DID - invocation: UnknownLink - blob: Blob -} - -export interface BlobAddOutput - extends Omit {} - export interface StoreAddInput { space: DID link: UnknownLink diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts new file mode 100644 index 000000000..7a543b031 --- /dev/null +++ b/packages/upload-api/src/types/blob.ts @@ -0,0 +1,75 @@ +import type { + UnknownLink, + Invocation, + Result, + Failure, + DID, +} from '@ucanto/interface' +import { + BlobMultihash, + BlobListItem, + BlobRemoveSuccess, +} from '@web3-storage/capabilities/types' + +import { + RecordKeyConflict, + RecordNotFound, + ListOptions, + ListResponse, +} from '../types.js' +import { Storage } from './storage.js' + +export type TasksStorage = Storage + +export interface AllocationsStorage { + exists: ( + space: DID, + blobMultihash: BlobMultihash + ) => Promise> + /** Inserts an item in the table if it does not already exist. */ + insert: ( + item: BlobAddInput + ) => Promise> + /** Removes an item from the table but fails if the item does not exist. */ + remove: ( + space: DID, + blobMultihash: BlobMultihash + ) => Promise> + list: ( + space: DID, + options?: ListOptions + ) => Promise, Failure>> +} + +export interface Blob { + content: BlobMultihash + size: number +} + +export interface BlobAddInput { + space: DID + invocation: UnknownLink + blob: Blob +} + +export interface BlobAddOutput + extends Omit {} + +export interface BlobsStorage { + has: (content: BlobMultihash) => Promise> + createUploadUrl: ( + content: BlobMultihash, + size: number + ) => Promise< + Result< + { + url: URL + headers: { + 'x-amz-checksum-sha256': string + 'content-length': string + } & Record + }, + Failure + > + > +} diff --git a/packages/upload-api/src/types/service.ts b/packages/upload-api/src/types/service.ts new file mode 100644 index 000000000..066d50438 --- /dev/null +++ b/packages/upload-api/src/types/service.ts @@ -0,0 +1,4 @@ +import type { UnknownLink, Receipt } from '@ucanto/interface' +import { Storage } from './storage.js' + +export type ReceiptsStorage = Storage diff --git a/packages/upload-api/src/types/storage.ts b/packages/upload-api/src/types/storage.ts new file mode 100644 index 000000000..bc3b18ff2 --- /dev/null +++ b/packages/upload-api/src/types/storage.ts @@ -0,0 +1,34 @@ +import type { Unit, Result } from '@ucanto/interface' + +export interface Storage { + /** + * Puts a record in the store. + */ + put: (record: Rec) => Promise> + /** + * Gets a record from the store. + */ + get: (key: RecKey) => Promise> + /** + * Determine if a record already exists in the store for the given key. + */ + has: (key: RecKey) => Promise> +} + +export type StoragePutError = StorageOperationError | EncodeRecordFailed +export type StorageGetError = + | StorageOperationError + | EncodeRecordFailed + | RecordNotFound + +export interface StorageOperationError extends Error { + name: 'StorageOperationFailed' +} + +export interface RecordNotFound extends Error { + name: 'RecordNotFound' +} + +export interface EncodeRecordFailed extends Error { + name: 'EncodeRecordFailed' +} diff --git a/packages/upload-api/src/ucan.js b/packages/upload-api/src/ucan.js index 9bf9b14a6..47bc64df8 100644 --- a/packages/upload-api/src/ucan.js +++ b/packages/upload-api/src/ucan.js @@ -1,4 +1,5 @@ import { ucanRevokeProvider } from './ucan/revoke.js' +import { ucanConcludeProvider } from './ucan/conclude.js' import * as API from './types.js' /** @@ -6,6 +7,7 @@ import * as API from './types.js' */ export const createService = (context) => { return { + conclude: ucanConcludeProvider(context), revoke: ucanRevokeProvider(context), } } diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js new file mode 100644 index 000000000..a3d94a237 --- /dev/null +++ b/packages/upload-api/src/ucan/conclude.js @@ -0,0 +1,18 @@ +import { provide } from '@ucanto/server' +import { conclude } from '@web3-storage/capabilities/ucan' +import * as API from '../types.js' + +/** + * @param {API.ConcludeServiceContext} context + * @returns {API.ServiceMethod} + */ +export const ucanConcludeProvider = ({ receiptsStorage }) => + provide(conclude, async ({ capability, invocation }) => { + // TODO: Store receipt + + // TODO: Schedule accept (temporary simple hack) + + return { + ok: { time: Date.now() }, + } + }) diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index acee6a788..e4c126ac8 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -14,125 +14,95 @@ import { BlobItemSizeExceededName } from '../../src/blob/lib.js' * @type {API.Tests} */ export const test = { - 'blob/add schedules allocation and returns effects for allocation and accept': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // invoke `blob/add` - const invocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + 'blob/add schedules allocation and returns effects for allocation and accept': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // invoke `blob/add` + const invocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [proof], - }) - const blobAdd = await invocation.execute(connection) - if (!blobAdd.out.ok) { - console.log('out error') - throw new Error('invocation failed', { cause: blobAdd }) - } - - assert.ok(blobAdd.out.ok.claim) - assert.ok(blobAdd.fx.fork.length) - assert.ok(blobAdd.fx.join) - assert.ok(blobAdd.out.ok.claim['await/ok'].equals(blobAdd.fx.join)) - - // validate scheduled task ran - // await deferredSchedule.promise - // assert.equal(scheduledTasks.length, 1) - // const [blobAllocateInvocation] = scheduledTasks - // assert.equal(blobAllocateInvocation.can, BlobCapabilities.allocate.can) - // assert.equal(blobAllocateInvocation.nb.space, spaceDid) - // assert.equal(blobAllocateInvocation.nb.blob.size, size) - // assert.ok(equals(blobAllocateInvocation.nb.blob.content, content)) - }, - 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // invoke `blob/add` - const invocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size: Number.MAX_SAFE_INTEGER + proofs: [proof], + }) + const blobAdd = await invocation.execute(connection) + if (!blobAdd.out.ok) { + console.log('out error') + throw new Error('invocation failed', { cause: blobAdd }) + } + + assert.ok(blobAdd.out.ok.claim) + assert.equal(blobAdd.fx.fork.length, 2) + assert.ok(blobAdd.fx.join) + assert.ok(blobAdd.out.ok.claim['await/ok'].equals(blobAdd.fx.join)) + + // Validate `http/put` invocation stored + // TODO, needs receipt to include those bytes + + // validate scheduled task ran and has receipt inlined + // const [blobAllocateInvocation] = scheduledTasks + // assert.equal(blobAllocateInvocation.can, BlobCapabilities.allocate.can) + // assert.equal(blobAllocateInvocation.nb.space, spaceDid) + // assert.equal(blobAllocateInvocation.nb.blob.size, size) + // assert.ok(equals(blobAllocateInvocation.nb.blob.content, content)) + }, + 'blob/add fails when a blob with size bigger than maximum size is added': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // invoke `blob/add` + const invocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size: Number.MAX_SAFE_INTEGER, + }, }, - }, - proofs: [proof], - }) - const blobAdd = await invocation.execute(connection) - if (!blobAdd.out.error) { - throw new Error('invocation should have failed') - } - assert.ok(blobAdd.out.error, 'invocation should have failed') - assert.equal(blobAdd.out.error.name, BlobItemSizeExceededName) - }, - 'skip blob/add fails when allocate task cannot be scheduled': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // invoke `blob/add` - const invocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size - }, - }, - proofs: [proof], - }) - const blobAdd = await invocation.execute(connection) - if (!blobAdd.out.error) { - throw new Error('invocation should have failed') - } - assert.ok(blobAdd.out.error, 'invocation should have failed') - assert.ok(blobAdd.out.error.message.includes(BlobCapabilities.allocate.can)) - assert.equal(blobAdd.out.error.name, 'Error') - }, - 'blob/allocate allocates to space and returns presigned url': async (assert, context) => { + proofs: [proof], + }) + const blobAdd = await invocation.execute(connection) + if (!blobAdd.out.error) { + throw new Error('invocation should have failed') + } + assert.ok(blobAdd.out.error, 'invocation should have failed') + assert.equal(blobAdd.out.error.name, BlobItemSizeExceededName) + }, + 'blob/allocate allocates to space and returns presigned url': async ( + assert, + context + ) => { const { proof, spaceDid } = await registerSpace(alice, context) // prepare data @@ -156,7 +126,7 @@ export const test = { nb: { blob: { content, - size + size, }, }, proofs: [proof], @@ -170,7 +140,7 @@ export const test = { nb: { blob: { content, - size + size, }, cause: (await blobAddInvocation.delegate()).cid, space: spaceDid, @@ -187,13 +157,18 @@ export const test = { assert.ok(blobAllocate.out.ok.address) assert.ok(blobAllocate.out.ok.address?.headers) assert.ok(blobAllocate.out.ok.address?.url) - assert.equal(blobAllocate.out.ok.address?.headers?.['content-length'], String(size)) + assert.equal( + blobAllocate.out.ok.address?.headers?.['content-length'], + String(size) + ) assert.deepEqual( blobAllocate.out.ok.address?.headers?.['x-amz-checksum-sha256'], base64pad.baseEncode(digest) ) - const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) if (!url) { throw new Error('Expected presigned url in response') } @@ -206,7 +181,7 @@ export const test = { ) // Validate allocation state - const spaceAllocations = await context.allocationStorage.list(spaceDid) + const spaceAllocations = await context.allocationsStorage.list(spaceDid) assert.ok(spaceAllocations.ok) assert.equal(spaceAllocations.ok?.size, 1) const allocatedEntry = spaceAllocations.ok?.results[0] @@ -226,325 +201,345 @@ export const test = { assert.equal(goodPut.status, 200, await goodPut.text()) }, - 'blob/allocate does not allocate more space to already allocated content': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + 'blob/allocate does not allocate more space to already allocated content': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - - // second blob allocate invocation - const secondBlobAllocate = await serviceBlobAllocate.execute(connection) - if (!secondBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: secondBlobAllocate }) - } - - // Validate response - assert.equal(secondBlobAllocate.out.ok.size, 0) - assert.ok(!!blobAllocate.out.ok.address) - }, - 'blob/allocate can allocate to different space after write to one space': async (assert, context) => { - const { proof: aliceProof, spaceDid: aliceSpaceDid } = await registerSpace(alice, context) - const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( - bob, - context, - 'bob' - ) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocations - const aliceBlobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: aliceSpaceDid, - nb: { - blob: { - content, - size + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + + // second blob allocate invocation + const secondBlobAllocate = await serviceBlobAllocate.execute(connection) + if (!secondBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: secondBlobAllocate }) + } + + // Validate response + assert.equal(secondBlobAllocate.out.ok.size, 0) + assert.ok(!!blobAllocate.out.ok.address) + }, + 'blob/allocate can allocate to different space after write to one space': + async (assert, context) => { + const { proof: aliceProof, spaceDid: aliceSpaceDid } = + await registerSpace(alice, context) + const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( + bob, + context, + 'bob' + ) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocations + const aliceBlobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [aliceProof], - }) - const bobBlobAddInvocation = BlobCapabilities.add.invoke({ - issuer: bob, - audience: context.id, - with: bobSpaceDid, - nb: { - blob: { - content, - size + proofs: [aliceProof], + }) + const bobBlobAddInvocation = BlobCapabilities.add.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [bobProof], - }) - - // invoke `service/blob/allocate` capabilities on alice space - const aliceServiceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: aliceSpaceDid, - nb: { - blob: { - content, - size + proofs: [bobProof], + }) + + // invoke `service/blob/allocate` capabilities on alice space + const aliceServiceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await aliceBlobAddInvocation.delegate()).cid, + space: aliceSpaceDid, }, - cause: (await aliceBlobAddInvocation.delegate()).cid, - space: aliceSpaceDid, - }, - proofs: [aliceProof], - }) - const aliceBlobAllocate = await aliceServiceBlobAllocate.execute(connection) - if (!aliceBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: aliceBlobAllocate }) - } - // there is address to write - assert.ok(aliceBlobAllocate.out.ok.address) - assert.equal(aliceBlobAllocate.out.ok.size, size) - - // write to presigned url - const url = aliceBlobAllocate.out.ok.address?.url && new URL(aliceBlobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const goodPut = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: data, - headers: aliceBlobAllocate.out.ok.address?.headers, - }) - - assert.equal(goodPut.status, 200, await goodPut.text()) - - // invoke `service/blob/allocate` capabilities on bob space - const bobServiceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: bob, - audience: context.id, - with: bobSpaceDid, - nb: { - blob: { - content, - size + proofs: [aliceProof], + }) + const aliceBlobAllocate = await aliceServiceBlobAllocate.execute( + connection + ) + if (!aliceBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: aliceBlobAllocate }) + } + // there is address to write + assert.ok(aliceBlobAllocate.out.ok.address) + assert.equal(aliceBlobAllocate.out.ok.size, size) + + // write to presigned url + const url = + aliceBlobAllocate.out.ok.address?.url && + new URL(aliceBlobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const goodPut = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: aliceBlobAllocate.out.ok.address?.headers, + }) + + assert.equal(goodPut.status, 200, await goodPut.text()) + + // invoke `service/blob/allocate` capabilities on bob space + const bobServiceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await bobBlobAddInvocation.delegate()).cid, + space: bobSpaceDid, }, - cause: (await bobBlobAddInvocation.delegate()).cid, - space: bobSpaceDid, - }, - proofs: [bobProof], - }) - const bobBlobAllocate = await bobServiceBlobAllocate.execute(connection) - if (!bobBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: bobBlobAllocate }) - } - // there is no address to write - assert.ok(!bobBlobAllocate.out.ok.address) - assert.equal(bobBlobAllocate.out.ok.size, size) - - // Validate allocation state - const aliceSpaceAllocations = await context.allocationStorage.list(aliceSpaceDid) - assert.ok(aliceSpaceAllocations.ok) - assert.equal(aliceSpaceAllocations.ok?.size, 1) - - const bobSpaceAllocations = await context.allocationStorage.list(bobSpaceDid) - assert.ok(bobSpaceAllocations.ok) - assert.equal(bobSpaceAllocations.ok?.size, 1) - }, - 'blob/allocate creates presigned url that can only PUT a payload with right length': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + proofs: [bobProof], + }) + const bobBlobAllocate = await bobServiceBlobAllocate.execute(connection) + if (!bobBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: bobBlobAllocate }) + } + // there is no address to write + assert.ok(!bobBlobAllocate.out.ok.address) + assert.equal(bobBlobAllocate.out.ok.size, size) + + // Validate allocation state + const aliceSpaceAllocations = await context.allocationsStorage.list( + aliceSpaceDid + ) + assert.ok(aliceSpaceAllocations.ok) + assert.equal(aliceSpaceAllocations.ok?.size, 1) + + const bobSpaceAllocations = await context.allocationsStorage.list( + bobSpaceDid + ) + assert.ok(bobSpaceAllocations.ok) + assert.equal(bobSpaceAllocations.ok?.size, 1) + }, + 'blob/allocate creates presigned url that can only PUT a payload with right length': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - // there is address to write - assert.ok(blobAllocate.out.ok.address) - assert.equal(blobAllocate.out.ok.size, size) - - // write to presigned url - const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const contentLengthFailSignature = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: longer, - headers: { - ...blobAllocate.out.ok.address?.headers, - 'content-length': longer.byteLength.toString(10), - }, - }) - - assert.equal( - contentLengthFailSignature.status >= 400, - true, - 'should fail to upload as content-length differs from that used to sign the url' - ) - }, - 'blob/allocate creates presigned url that can only PUT a payload with exact bytes': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const other = new Uint8Array([10, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const content = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + // there is address to write + assert.ok(blobAllocate.out.ok.address) + assert.equal(blobAllocate.out.ok.size, size) + + // write to presigned url + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const contentLengthFailSignature = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: longer, + headers: { + ...blobAllocate.out.ok.address?.headers, + 'content-length': longer.byteLength.toString(10), }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - content, - size + }) + + assert.equal( + contentLengthFailSignature.status >= 400, + true, + 'should fail to upload as content-length differs from that used to sign the url' + ) + }, + 'blob/allocate creates presigned url that can only PUT a payload with exact bytes': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const other = new Uint8Array([10, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - // there is address to write - assert.ok(blobAllocate.out.ok.address) - assert.equal(blobAllocate.out.ok.size, size) - - // write to presigned url - const url = blobAllocate.out.ok.address?.url && new URL(blobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const failChecksum = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: other, - headers: blobAllocate.out.ok.address?.headers, - }) - - assert.equal( - failChecksum.status, - 400, - 'should fail to upload any other data.' - ) - }, - 'blob/allocate disallowed if invocation fails access verification': async (assert, context) => { + proofs: [proof], + }) + + // invoke `service/blob/allocate` + const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + // there is address to write + assert.ok(blobAllocate.out.ok.address) + assert.equal(blobAllocate.out.ok.size, size) + + // write to presigned url + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const failChecksum = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: other, + headers: blobAllocate.out.ok.address?.headers, + }) + + assert.equal( + failChecksum.status, + 400, + 'should fail to upload any other data.' + ) + }, + 'blob/allocate disallowed if invocation fails access verification': async ( + assert, + context + ) => { const { proof, space, spaceDid } = await createSpace(alice) // prepare data @@ -567,7 +562,7 @@ export const test = { nb: { blob: { content, - size + size, }, }, proofs: [proof], @@ -581,7 +576,7 @@ export const test = { nb: { blob: { content, - size + size, }, cause: (await blobAddInvocation.delegate()).cid, space: spaceDid, diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index 6176b318b..1926fb63a 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -2,28 +2,18 @@ import * as Signer from '@ucanto/principal/ed25519' import { getConnection, getMockService, - getStoreImplementations, - getQueueImplementations, + getStoreImplementations as getFilecoinStoreImplementations, + getQueueImplementations as getFilecoinQueueImplementations, } from '@web3-storage/filecoin-api/test/context/service' -import { AllocationStorage } from '../storage/allocation-storage.js' -import { BlobStorage } from '../storage/blob-storage.js' +import { BlobsStorage } from '../storage/blobs-storage.js' import { CarStoreBucket } from '../storage/car-store-bucket.js' -import { StoreTable } from '../storage/store-table.js' -import { UploadTable } from '../storage/upload-table.js' -import { DudewhereBucket } from '../storage/dude-where-bucket.js' -import { ProvisionsStorage } from '../storage/provisions-storage.js' -import { DelegationsStorage } from '../storage/delegations-storage.js' -import { RateLimitsStorage } from '../storage/rate-limits-storage.js' -import { RevocationsStorage } from '../storage/revocations-storage.js' import * as Email from '../../src/utils/email.js' import { create as createRevocationChecker } from '../../src/utils/revocation.js' import { createServer, connect } from '../../src/lib.js' import * as Types from '../../src/types.js' import * as TestTypes from '../types.js' import { confirmConfirmationUrl } from './utils.js' -import { PlansStorage } from '../storage/plans-storage.js' -import { UsageStorage } from '../storage/usage-storage.js' -import { SubscriptionsStorage } from '../storage/subscriptions-storage.js' +import { getServiceStorageImplementations } from '../storage/index.js' /** * @param {object} options @@ -37,19 +27,6 @@ export const createContext = async ( options = { requirePaymentPlan: false } ) => { const requirePaymentPlan = options.requirePaymentPlan - const storeTable = new StoreTable() - const allocationStorage = new AllocationStorage() - const uploadTable = new UploadTable() - const blobStorage = await BlobStorage.activate(options) - const carStoreBucket = await CarStoreBucket.activate(options) - const dudewhereBucket = new DudewhereBucket() - const revocationsStorage = new RevocationsStorage() - const plansStorage = new PlansStorage() - const usageStorage = new UsageStorage(storeTable) - const provisionsStorage = new ProvisionsStorage(options.providers) - const subscriptionsStorage = new SubscriptionsStorage(provisionsStorage) - const delegationsStorage = new DelegationsStorage() - const rateLimitsStorage = new RateLimitsStorage() const signer = await Signer.generate() const aggregatorSigner = await Signer.generate() const dealTrackerSigner = await Signer.generate() @@ -61,14 +38,16 @@ export const createContext = async ( service ).connection + const serviceStores = await getServiceStorageImplementations(options) + /** @type {Map} */ const queuedMessages = new Map() const { storefront: { filecoinSubmitQueue, pieceOfferQueue }, - } = getQueueImplementations(queuedMessages) + } = getFilecoinQueueImplementations(queuedMessages) const { storefront: { pieceStore, receiptStore, taskStore }, - } = getStoreImplementations() + } = getFilecoinStoreImplementations() const email = Email.debug() /** @type { import('../../src/types.js').UcantoServerContext } */ @@ -77,14 +56,13 @@ export const createContext = async ( aggregatorId: aggregatorSigner, signer: id, email, + requirePaymentPlan, url: new URL('http://localhost:8787'), - provisionsStorage, - subscriptionsStorage, - delegationsStorage, - rateLimitsStorage, - plansStorage, - usageStorage, - revocationsStorage, + ...serviceStores, + getServiceConnection: () => connection, + ...createRevocationChecker({ + revocationsStorage: serviceStores.revocationsStorage, + }), errorReporter: { catch(error) { if (options.assert) { @@ -94,19 +72,13 @@ export const createContext = async ( } }, }, + // Filecoin maxUploadSize: 5_000_000_000, - storeTable, - allocationStorage, - uploadTable, - carStoreBucket, - blobStorage, - dudewhereBucket, filecoinSubmitQueue, pieceOfferQueue, pieceStore, receiptStore, taskStore, - requirePaymentPlan, dealTrackerService: { connection: dealTrackerConnection, invocationConfig: { @@ -115,8 +87,6 @@ export const createContext = async ( audience: dealTrackerSigner, }, }, - getServiceConnection: () => connection, - ...createRevocationChecker({ revocationsStorage }), } const connection = connect({ @@ -144,8 +114,8 @@ export const cleanupContext = async (context) => { const carStoreBucket = context.carStoreBucket await carStoreBucket.deactivate() - /** @type {BlobStorage & { deactivate: () => Promise }}} */ + /** @type {BlobsStorage & { deactivate: () => Promise }}} */ // @ts-ignore type misses S3 bucket properties like accessKey - const blobStorage = context.blobStorage - await blobStorage.deactivate() + const blobsStorage = context.blobsStorage + await blobsStorage.deactivate() } diff --git a/packages/upload-api/test/storage/allocation-storage.js b/packages/upload-api/test/storage/allocations-storage.js similarity index 88% rename from packages/upload-api/test/storage/allocation-storage.js rename to packages/upload-api/test/storage/allocations-storage.js index 7d30d0a46..d6982bd59 100644 --- a/packages/upload-api/test/storage/allocation-storage.js +++ b/packages/upload-api/test/storage/allocations-storage.js @@ -2,9 +2,9 @@ import * as Types from '../../src/types.js' import { equals } from 'uint8arrays/equals' /** - * @implements {Types.AllocationStorage} + * @implements {Types.AllocationsStorage} */ -export class AllocationStorage { +export class AllocationsStorage { constructor() { /** @type {(Types.BlobAddInput & Types.BlobListItem)[]} */ this.items = [] @@ -12,7 +12,7 @@ export class AllocationStorage { /** * @param {Types.BlobAddInput} input - * @returns {ReturnType} + * @returns {ReturnType} */ async insert({ space, invocation, ...output }) { if ( @@ -36,7 +36,7 @@ export class AllocationStorage { /** * @param {Types.DID} space * @param {Uint8Array} blobMultihash - * @returns {ReturnType} + * @returns {ReturnType} */ async exists(space, blobMultihash) { const item = this.items.find( @@ -48,7 +48,7 @@ export class AllocationStorage { /** * @param {Types.DID} space * @param {Uint8Array} blobMultihash - * @returns {ReturnType} + * @returns {ReturnType} */ async remove(space, blobMultihash) { const item = this.items.find( @@ -68,7 +68,7 @@ export class AllocationStorage { /** * @param {Types.DID} space * @param {Types.ListOptions} options - * @returns {ReturnType} + * @returns {ReturnType} */ async list( space, diff --git a/packages/upload-api/test/storage/blob-storage.js b/packages/upload-api/test/storage/blobs-storage.js similarity index 97% rename from packages/upload-api/test/storage/blob-storage.js rename to packages/upload-api/test/storage/blobs-storage.js index 807d029d7..51135e809 100644 --- a/packages/upload-api/test/storage/blob-storage.js +++ b/packages/upload-api/test/storage/blobs-storage.js @@ -7,9 +7,9 @@ import { base58btc } from 'multiformats/bases/base58' import { sha256 } from 'multiformats/hashes/sha2' /** - * @implements {Types.BlobStorage} + * @implements {Types.BlobsStorage} */ -export class BlobStorage { +export class BlobsStorage { /** * @param {Types.CarStoreBucketOptions & {http?: import('http')}} options */ @@ -50,14 +50,14 @@ export class BlobStorage { const port = server.address().port const url = new URL(`http://localhost:${port}`) - return new BlobStorage({ + return new BlobsStorage({ ...options, content, url, server, }) } else { - return new BlobStorage({ + return new BlobsStorage({ ...options, content, url: new URL(`http://localhost:8989`), diff --git a/packages/upload-api/test/storage/index.js b/packages/upload-api/test/storage/index.js new file mode 100644 index 000000000..89d3fc51c --- /dev/null +++ b/packages/upload-api/test/storage/index.js @@ -0,0 +1,58 @@ +import { AllocationsStorage } from './allocations-storage.js' +import { BlobsStorage } from './blobs-storage.js' +import { CarStoreBucket } from './car-store-bucket.js' +import { StoreTable } from './store-table.js' +import { UploadTable } from './upload-table.js' +import { DudewhereBucket } from './dude-where-bucket.js' +import { ProvisionsStorage } from './provisions-storage.js' +import { DelegationsStorage } from './delegations-storage.js' +import { RateLimitsStorage } from './rate-limits-storage.js' +import { RevocationsStorage } from './revocations-storage.js' +import { PlansStorage } from './plans-storage.js' +import { UsageStorage } from './usage-storage.js' +import { SubscriptionsStorage } from './subscriptions-storage.js' +import { TasksStorage } from './tasks-storage.js' +import { ReceiptsStorage } from './receipts-storage.js' + +/** + * @param {object} options + * @param {string[]} [options.providers] + * @param {boolean} [options.requirePaymentPlan] + * @param {import('http')} [options.http] + * @param {{fail(error:unknown): unknown}} [options.assert] + */ +export async function getServiceStorageImplementations(options) { + const storeTable = new StoreTable() + const allocationsStorage = new AllocationsStorage() + const uploadTable = new UploadTable() + const blobsStorage = await BlobsStorage.activate(options) + const carStoreBucket = await CarStoreBucket.activate(options) + const dudewhereBucket = new DudewhereBucket() + const revocationsStorage = new RevocationsStorage() + const plansStorage = new PlansStorage() + const usageStorage = new UsageStorage(storeTable) + const provisionsStorage = new ProvisionsStorage(options.providers) + const subscriptionsStorage = new SubscriptionsStorage(provisionsStorage) + const delegationsStorage = new DelegationsStorage() + const rateLimitsStorage = new RateLimitsStorage() + const tasksStorage = new TasksStorage() + const receiptsStorage = new ReceiptsStorage() + + return { + storeTable, + allocationsStorage, + uploadTable, + blobsStorage, + carStoreBucket, + dudewhereBucket, + revocationsStorage, + plansStorage, + usageStorage, + provisionsStorage, + subscriptionsStorage, + delegationsStorage, + rateLimitsStorage, + tasksStorage, + receiptsStorage, + } +} diff --git a/packages/upload-api/test/storage/receipts-storage.js b/packages/upload-api/test/storage/receipts-storage.js new file mode 100644 index 000000000..91f876e0b --- /dev/null +++ b/packages/upload-api/test/storage/receipts-storage.js @@ -0,0 +1,64 @@ +import * as API from '../../src/types.js' + +import { RecordNotFound } from '../../src/errors.js' + +/** + * @typedef {import('../../src/types/storage.js').StorageGetError} StorageGetError + * @typedef {import('../../src/types/storage.js').StoragePutError} StoragePutError + * @typedef {import('@ucanto/interface').UnknownLink} UnknownLink + * @typedef {import('@ucanto/interface').Receipt} Receipt + */ + +/** + * @implements {API.ReceiptsStorage} + */ +export class ReceiptsStorage { + constructor() { + /** @type {Map} */ + this.items = new Map() + } + + /** + * @param {Receipt} record + * @returns {Promise>} + */ + async put(record) { + this.items.set(record.ran.link(), record) + + return Promise.resolve({ + ok: {}, + }) + } + + /** + * @param {UnknownLink} link + * @returns {Promise>} + */ + async get(link) { + const record = this.items.get(link) + if (!record) { + return { + error: new RecordNotFound('not found'), + } + } + return { + ok: record, + } + } + + /** + * @param {UnknownLink} link + * @returns {Promise>} + */ + async has(link) { + const record = this.items.get(link) + if (!record) { + return { + ok: false, + } + } + return { + ok: Boolean(record), + } + } +} diff --git a/packages/upload-api/test/storage/tasks-storage.js b/packages/upload-api/test/storage/tasks-storage.js new file mode 100644 index 000000000..9edb62347 --- /dev/null +++ b/packages/upload-api/test/storage/tasks-storage.js @@ -0,0 +1,64 @@ +import * as API from '../../src/types.js' + +import { RecordNotFound } from '../../src/errors.js' + +/** + * @typedef {import('../../src/types/storage.js').StorageGetError} StorageGetError + * @typedef {import('../../src/types/storage.js').StoragePutError} StoragePutError + * @typedef {import('@ucanto/interface').UnknownLink} UnknownLink + * @typedef {import('@ucanto/interface').Invocation} Invocation + */ + +/** + * @implements {API.TasksStorage} + */ +export class TasksStorage { + constructor() { + /** @type {Map} */ + this.items = new Map() + } + + /** + * @param {Invocation} record + * @returns {Promise>} + */ + async put(record) { + this.items.set(record.cid, record) + + return Promise.resolve({ + ok: {}, + }) + } + + /** + * @param {UnknownLink} link + * @returns {Promise>} + */ + async get(link) { + const record = this.items.get(link) + if (!record) { + return { + error: new RecordNotFound('not found'), + } + } + return { + ok: record, + } + } + + /** + * @param {UnknownLink} link + * @returns {Promise>} + */ + async has(link) { + const record = this.items.get(link) + if (!record) { + return { + ok: false, + } + } + return { + ok: Boolean(record), + } + } +} From e09310bb4b986ed47d84f6401af5c616724c84f7 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 4 Apr 2024 14:00:16 +0200 Subject: [PATCH 03/27] fix: upgrade ucanto libs --- packages/capabilities/src/ucan.js | 62 ++++++++++--------- packages/upload-api/src/blob/add.js | 32 ++++++++-- packages/upload-api/src/ucan/conclude.js | 22 ++++++- packages/upload-api/test/handlers/blob.js | 49 +++++++++++---- .../upload-api/test/storage/tasks-storage.js | 8 +-- 5 files changed, 121 insertions(+), 52 deletions(-) diff --git a/packages/capabilities/src/ucan.js b/packages/capabilities/src/ucan.js index 83ce637fa..c703f2f97 100644 --- a/packages/capabilities/src/ucan.js +++ b/packages/capabilities/src/ucan.js @@ -91,40 +91,42 @@ export const conclude = capability({ with: Schema.did(), // TODO: Should this just have bytes? nb: Schema.struct({ - /** - * A link to the UCAN invocation that this receipt is for. - */ - ran: UCANLink, - /** - * The value output of the invocation in Result format. - */ - out: Schema.unknown(), - /** - * Tasks that the invocation would like to enqueue. - */ - next: Schema.array(UCANLink), - /** - * Additional data about the receipt - */ - meta: Schema.unknown(), - /** - * The UTC Unix timestamp at which the Receipt was issued - */ - time: Schema.integer(), + bytes: Schema.Bytes, + // /** + // * A link to the UCAN invocation that this receipt is for. + // */ + // ran: UCANLink, + // /** + // * The value output of the invocation in Result format. + // */ + // out: Schema.unknown(), + // /** + // * Tasks that the invocation would like to enqueue. + // */ + // next: Schema.array(UCANLink), + // /** + // * Additional data about the receipt + // */ + // meta: Schema.unknown(), + // /** + // * The UTC Unix timestamp at which the Receipt was issued + // */ + // time: Schema.integer(), }), derives: (claim, from) => // With field MUST be the same and(equalWith(claim, from)) ?? - // invocation MUST be the same - and(checkLink(claim.nb.ran, from.nb.ran, 'nb.ran')) ?? - // value output MUST be the same - and(equal(claim.nb.out, from.nb.out, 'nb.out')) ?? - // tasks to enqueue MUST be the same - and(equal(claim.nb.next, from.nb.next, 'nb.next')) ?? - // additional data MUST be the same - and(equal(claim.nb.meta, from.nb.meta, 'nb.meta')) ?? - // the receipt issue time MUST be the same - equal(claim.nb.time, from.nb.time, 'nb.time'), + equal(claim.nb.bytes, from.nb.bytes, 'nb.bytes'), + // // invocation MUST be the same + // and(checkLink(claim.nb.ran, from.nb.ran, 'nb.ran')) ?? + // // value output MUST be the same + // and(equal(claim.nb.out, from.nb.out, 'nb.out')) ?? + // // tasks to enqueue MUST be the same + // and(equal(claim.nb.next, from.nb.next, 'nb.next')) ?? + // // additional data MUST be the same + // and(equal(claim.nb.meta, from.nb.meta, 'nb.meta')) ?? + // // the receipt issue time MUST be the same + // equal(claim.nb.time, from.nb.time, 'nb.time'), }) /** diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index f46e5e06d..fbd544e07 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -1,6 +1,9 @@ import * as Server from '@ucanto/server' +import { Message } from '@ucanto/core' import { ed25519 } from '@ucanto/principal' +import { CAR } from '@ucanto/transport' import * as Blob from '@web3-storage/capabilities/blob' +import * as UCAN from '@web3-storage/capabilities/ucan' import * as API from '../types.js' import { BlobItemSizeExceeded } from './lib.js' @@ -103,6 +106,7 @@ export function blobAddProvider(context) { space, blob.content ) + let allocateUcanConcludefx if (!allocatedExistsRes.ok) { // Execute allocate invocation const allocateRes = await blobAllocate.execute(getServiceConnection()) @@ -111,6 +115,17 @@ export function blobAddProvider(context) { error: allocateRes.out.error, } } + const message = await Message.build({ receipts: [allocateRes] }) + const messageCar = await CAR.outbound.encode(message) + allocateUcanConcludefx = await UCAN.conclude.invoke({ + issuer: id, + audience: id, + with: id.toDIDKey(), + nb: { + bytes: messageCar.body + }, + expiration: Infinity, + }).delegate() } /** @type {API.OkBuilder} */ @@ -119,11 +134,20 @@ export function blobAddProvider(context) { 'await/ok': acceptfx.link(), }, }) - // TODO: not pass links, but delegation + // Add allocate receipt if allocate was executed + if (allocateUcanConcludefx) { + // TODO: perhaps if we allocated we need to get and write previous receipt? + return result + .fork(allocatefx) + .fork(allocateUcanConcludefx) + .fork(putfx) + .join(acceptfx) + } + return result - .fork(allocatefx.link()) - .fork(putfx.link()) - .join(acceptfx.link()) + .fork(allocatefx) + .fork(putfx) + .join(acceptfx) }, }) } diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index a3d94a237..e88ba6f18 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -1,4 +1,6 @@ import { provide } from '@ucanto/server' +import { Message } from '@ucanto/core' +import { CAR } from '@ucanto/transport' import { conclude } from '@web3-storage/capabilities/ucan' import * as API from '../types.js' @@ -7,8 +9,24 @@ import * as API from '../types.js' * @returns {API.ServiceMethod} */ export const ucanConcludeProvider = ({ receiptsStorage }) => - provide(conclude, async ({ capability, invocation }) => { - // TODO: Store receipt + provide(conclude, async ({ capability }) => { + const messageCar = CAR.codec.decode(capability.nb.bytes) + const message = Message.view({ root: messageCar.roots[0].cid, store: messageCar.blocks }) + + // TODO: check number of receipts + const receiptKey = Array.from(message.receipts.keys())[0] + const receipt = message.receipts.get(receiptKey) + + if (!receipt) { + throw new Error('receipt should exist') + } + + const receiptPutRes = await receiptsStorage.put(receipt) + if (receiptPutRes.error) { + return { + error: receiptPutRes.error + } + } // TODO: Schedule accept (temporary simple hack) diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index e4c126ac8..a6d51afa6 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -1,8 +1,11 @@ import * as API from '../../src/types.js' import { Absentee } from '@ucanto/principal' +import { Message } from '@ucanto/core' +import { CAR } from '@ucanto/transport' import { equals } from 'uint8arrays' import { sha256 } from 'multiformats/hashes/sha2' import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import * as UCAN from '@web3-storage/capabilities/ucan' import { base64pad } from 'multiformats/bases/base64' import { provisionProvider } from '../helpers/utils.js' @@ -14,7 +17,7 @@ import { BlobItemSizeExceededName } from '../../src/blob/lib.js' * @type {API.Tests} */ export const test = { - 'blob/add schedules allocation and returns effects for allocation and accept': + 'blob/add executes allocation and returns effects for allocation, accept, put and allocate receipt': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -45,24 +48,46 @@ export const test = { }) const blobAdd = await invocation.execute(connection) if (!blobAdd.out.ok) { - console.log('out error') throw new Error('invocation failed', { cause: blobAdd }) } + // Validate receipt assert.ok(blobAdd.out.ok.claim) - assert.equal(blobAdd.fx.fork.length, 2) + assert.ok(blobAdd.out.ok.claim['await/ok'].equals(blobAdd.fx.join?.link())) assert.ok(blobAdd.fx.join) - assert.ok(blobAdd.out.ok.claim['await/ok'].equals(blobAdd.fx.join)) + + /** + * @type {import('@ucanto/interface').Invocation[]} + **/ + // @ts-expect-error read only effect + const forkInvocations = blobAdd.fx.fork + assert.equal(blobAdd.fx.fork.length, 3) + const allocatefx = forkInvocations.find(fork => fork.capabilities[0].can === BlobCapabilities.allocate.can) + const allocateUcanConcludefx = forkInvocations.find(fork => fork.capabilities[0].can === UCAN.conclude.can) + const putfx = forkInvocations.find(fork => fork.capabilities[0].can === BlobCapabilities.put.can) + if (!allocatefx || !allocateUcanConcludefx || !putfx) { + throw new Error('effects not provided') + } // Validate `http/put` invocation stored - // TODO, needs receipt to include those bytes - - // validate scheduled task ran and has receipt inlined - // const [blobAllocateInvocation] = scheduledTasks - // assert.equal(blobAllocateInvocation.can, BlobCapabilities.allocate.can) - // assert.equal(blobAllocateInvocation.nb.space, spaceDid) - // assert.equal(blobAllocateInvocation.nb.blob.size, size) - // assert.ok(equals(blobAllocateInvocation.nb.blob.content, content)) + const httpPutGetTask = await context.tasksStorage.get( + putfx.cid + ) + assert.ok(httpPutGetTask.ok) + + // validate scheduled allocate task ran an its receipt content + // @ts-expect-error object of type unknown + const messageCar = CAR.codec.decode(allocateUcanConcludefx.capabilities[0].nb.bytes) + const message = Message.view({ root: messageCar.roots[0].cid, store: messageCar.blocks }) + + const receiptKey = Array.from(message.receipts.keys())[0] + const receipt = message.receipts.get(receiptKey) + assert.ok(receipt?.out) + assert.ok(receipt?.out.ok) + // @ts-expect-error receipt out is unknown + assert.equal(receipt?.out.ok?.size, size) + // @ts-expect-error receipt out is unknown + assert.ok(receipt?.out.ok?.address) }, 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { diff --git a/packages/upload-api/test/storage/tasks-storage.js b/packages/upload-api/test/storage/tasks-storage.js index 9edb62347..43468a8bb 100644 --- a/packages/upload-api/test/storage/tasks-storage.js +++ b/packages/upload-api/test/storage/tasks-storage.js @@ -14,7 +14,7 @@ import { RecordNotFound } from '../../src/errors.js' */ export class TasksStorage { constructor() { - /** @type {Map} */ + /** @type {Map} */ this.items = new Map() } @@ -23,7 +23,7 @@ export class TasksStorage { * @returns {Promise>} */ async put(record) { - this.items.set(record.cid, record) + this.items.set(record.cid.toString(), record) return Promise.resolve({ ok: {}, @@ -35,7 +35,7 @@ export class TasksStorage { * @returns {Promise>} */ async get(link) { - const record = this.items.get(link) + const record = this.items.get(link.toString()) if (!record) { return { error: new RecordNotFound('not found'), @@ -51,7 +51,7 @@ export class TasksStorage { * @returns {Promise>} */ async has(link) { - const record = this.items.get(link) + const record = this.items.get(link.toString()) if (!record) { return { ok: false, From 71b4550a0227bc702a8cba6b8a4c2f0deda40f14 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 5 Apr 2024 13:08:08 +0200 Subject: [PATCH 04/27] fix: wire conclude and accept together --- packages/capabilities/src/blob.js | 7 +- packages/capabilities/src/utils.js | 2 +- packages/upload-api/package.json | 1 + packages/upload-api/src/blob/add.js | 143 +++++--- packages/upload-api/src/blob/allocate.js | 14 +- packages/upload-api/src/types.ts | 25 +- packages/upload-api/src/types/blob.ts | 9 + packages/upload-api/src/types/service.ts | 12 +- packages/upload-api/src/ucan/conclude.js | 56 +++- packages/upload-api/test/handlers/blob.js | 304 +++++++++++++++++- packages/upload-api/test/helpers/context.js | 6 + .../test/storage/allocations-storage.js | 15 + .../test/storage/receipts-storage.js | 8 +- pnpm-lock.yaml | 13 +- 14 files changed, 531 insertions(+), 84 deletions(-) diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 331dd893b..163781126 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -202,9 +202,9 @@ export const put = capability({ with: SpaceDID, nb: Schema.struct({ /** - * A multihash digest of the blob payload bytes, uniquely identifying blob. + * Blob to allocate on the space. */ - content: Schema.bytes(), + blob: blobStruct, /** * Blob to accept. */ @@ -221,7 +221,8 @@ export const put = capability({ }), derives: (claim, from) => { return ( - and(equalContent(claim, from)) || + and(equalWith(claim, from)) || + and(equalBlob(claim, from)) || and(equal(claim.nb.address?.url, from.nb.address, 'url')) || and(equal(claim.nb.address?.headers, from.nb.address, 'headers')) || ok({}) diff --git a/packages/capabilities/src/utils.js b/packages/capabilities/src/utils.js index 0db700c48..9e61fada9 100644 --- a/packages/capabilities/src/utils.js +++ b/packages/capabilities/src/utils.js @@ -88,7 +88,7 @@ export const equalLink = (claimed, delegated) => { } /** - * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"web3.storage/blob/allocate"|"web3.storage/blob/accept", Types.URI<'did:'>, {blob: { content: Uint8Array, size: number }}>} T + * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"web3.storage/blob/allocate"|"web3.storage/blob/accept"|"http/put", Types.URI<'did:'>, {blob: { content: Uint8Array, size: number }}>} T * @param {T} claimed * @param {T} delegated * @returns {Types.Result<{}, Types.Failure>} diff --git a/packages/upload-api/package.json b/packages/upload-api/package.json index 6c44985df..25cdfbb9f 100644 --- a/packages/upload-api/package.json +++ b/packages/upload-api/package.json @@ -196,6 +196,7 @@ "is-subset": "^0.1.1", "mocha": "^10.2.0", "one-webcrypto": "git://github.com/web3-storage/one-webcrypto", + "p-defer": "^4.0.1", "typescript": "5.2.2" }, "eslintConfig": { diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index fbd544e07..86f28b5ec 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -28,12 +28,14 @@ export function blobAddProvider(context) { Server.DID.parse(capability.with).did() ) + // Verify blob is within accept size if (blob.size > maxUploadSize) { return { error: new BlobItemSizeExceeded(maxUploadSize), } } + // Create shared subject for agent to issue `http/put` receipt const putSubject = await ed25519.derive(blob.content.slice(0, 32)) const facts = Object.entries(putSubject.toArchive().keys).map( ([key, value]) => ({ @@ -42,7 +44,7 @@ export function blobAddProvider(context) { }) ) - // Create effects for receipt + // Create web3.storage/blob/* invocations const blobAllocate = Blob.allocate.invoke({ issuer: id, audience: id, @@ -54,16 +56,6 @@ export function blobAddProvider(context) { }, expiration: Infinity, }) - const blobPut = Blob.put.invoke({ - issuer: putSubject, - audience: putSubject, - with: putSubject.toDIDKey(), - nb: { - content: blob.content, - }, - facts, - expiration: Infinity, - }) const blobAccept = Blob.accept.invoke({ issuer: id, audience: id, @@ -74,40 +66,39 @@ export function blobAddProvider(context) { }, expiration: Infinity, }) - const [allocatefx, putfx, acceptfx] = await Promise.all([ - // 1. System attempts to allocate memory in user space for the blob. + const [allocatefx, acceptfx] = await Promise.all([ blobAllocate.delegate(), - // 2. System requests user agent (or anyone really) to upload the content - // corresponding to the blob - // via HTTP PUT to given location. - blobPut.delegate(), - // 3. System will attempt to accept uploaded content that matches blob - // multihash and size. blobAccept.delegate(), ]) - // store `http/put` invocation - // TODO: store implementation - // const archiveDelegationRes = await putfx.archive() - // if (archiveDelegationRes.error) { - // return { - // error: archiveDelegationRes.error - // } - // } - const invocationPutRes = await tasksStorage.put(putfx) - if (invocationPutRes.error) { - return { - error: invocationPutRes.error, - } - } - // Schedule allocation if not allocated - const allocatedExistsRes = await allocationsStorage.exists( + // Get receipt for `blob/allocate` if available, or schedule invocation if not + const allocatedGetRes = await allocationsStorage.get( space, blob.content ) - let allocateUcanConcludefx - if (!allocatedExistsRes.ok) { + let blobAllocateReceipt + let blobAllocateOutAddress + // If already allocated, just get the allocate receipt + // and the addresses if still pending to receive blob + if (allocatedGetRes.ok) { + const receiptGet = await context.receiptsStorage.get(allocatefx.link()) + if (receiptGet.error) { + return receiptGet + } + blobAllocateReceipt = receiptGet.ok + + // Check if despite allocated, the blob is still not stored + const blobHasRes = await context.blobsStorage.has(blob.content) + if (blobHasRes.error) { + return blobHasRes + } else if (!blobHasRes.ok) { + // @ts-expect-error receipt type is unknown + blobAllocateOutAddress = blobAllocateReceipt.out.ok.address + } + } + // if not already allocated, schedule `blob/allocate` + else { // Execute allocate invocation const allocateRes = await blobAllocate.execute(getServiceConnection()) if (allocateRes.out.error) { @@ -115,38 +106,98 @@ export function blobAddProvider(context) { error: allocateRes.out.error, } } - const message = await Message.build({ receipts: [allocateRes] }) - const messageCar = await CAR.outbound.encode(message) - allocateUcanConcludefx = await UCAN.conclude.invoke({ + // If this is a new allocation, `http/put` effect should be returned with address + blobAllocateOutAddress = allocateRes.out.ok.address + blobAllocateReceipt = allocateRes + } + + // Create `blob/allocate` receipt invocation to inline as effect + const message = await Message.build({ receipts: [blobAllocateReceipt] }) + const messageCar = await CAR.outbound.encode(message) + const allocateUcanConcludefx = await UCAN.conclude + .invoke({ issuer: id, audience: id, with: id.toDIDKey(), nb: { - bytes: messageCar.body + bytes: messageCar.body, }, expiration: Infinity, - }).delegate() - } + }) + .delegate() + // Create result object /** @type {API.OkBuilder} */ const result = Server.ok({ claim: { 'await/ok': acceptfx.link(), }, }) + + // In case blob allocate provided an address to write + // the blob is still not stored + if (blobAllocateOutAddress) { + // Create effects for `blob/add` receipt + const blobPut = Blob.put.invoke({ + issuer: putSubject, + audience: putSubject, + with: putSubject.toDIDKey(), + nb: { + blob, + address: blobAllocateOutAddress + }, + facts, + expiration: Infinity, + }) + + const putfx = await blobPut.delegate() + + // store `http/put` invocation + // TODO: store implementation + // const archiveDelegationRes = await putfx.archive() + // if (archiveDelegationRes.error) { + // return { + // error: archiveDelegationRes.error + // } + // } + const invocationPutRes = await tasksStorage.put(putfx) + if (invocationPutRes.error) { + return { + error: invocationPutRes.error, + } + } + + return result + // 1. System attempts to allocate memory in user space for the blob. + .fork(allocatefx) + .fork(allocateUcanConcludefx) + // 2. System requests user agent (or anyone really) to upload the content + // corresponding to the blob + // via HTTP PUT to given location. + .fork(putfx) + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. + .join(acceptfx) + } + // Add allocate receipt if allocate was executed if (allocateUcanConcludefx) { - // TODO: perhaps if we allocated we need to get and write previous receipt? return result + // 1. System attempts to allocate memory in user space for the blob. .fork(allocatefx) .fork(allocateUcanConcludefx) - .fork(putfx) + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. .join(acceptfx) } + // Blob was already allocated and is already stored in the system return result + // 1. System allocated memory in user space for the blob. .fork(allocatefx) - .fork(putfx) + .fork(allocateUcanConcludefx) + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. .join(acceptfx) }, }) diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index 0693d3d3a..f61cbfa69 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -42,7 +42,7 @@ export function blobAllocateProvider(context) { } } - // If blob is stored, we can just allocate it to the space + // Check if blob already exists const hasBlob = await context.blobsStorage.has(blob.content) if (hasBlob.error) { return { @@ -59,14 +59,16 @@ export function blobAllocateProvider(context) { error: new Server.Failure('failed to provide presigned url'), } } + const address = { + url: createUploadUrl.ok.url.toString(), + headers: createUploadUrl.ok.headers, + } // Allocate in space, ignoring if already allocated const allocationInsert = await context.allocationsStorage.insert({ space, blob, invocation: cause, - // TODO: add write target here - // will the URL be enough to track? }) if (allocationInsert.error) { // if the insert failed with conflict then this item has already been @@ -81,6 +83,7 @@ export function blobAllocateProvider(context) { } } + // If blob is stored, we can just allocate it to the space if (hasBlob.ok) { return { ok: { size: blob.size }, @@ -90,10 +93,7 @@ export function blobAllocateProvider(context) { return { ok: { size: blob.size, - address: { - url: createUploadUrl.ok.url.toString(), - headers: createUploadUrl.ok.headers, - }, + address, }, } }) diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 3c932354f..ec9080616 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -138,6 +138,9 @@ import { CARLink, StoreGetSuccess, UploadGetSuccess, + UCANConclude, + UCANConcludeSuccess, + UCANConcludeFailure, UCANRevoke, UCANRevokeSuccess, UCANRevokeFailure, @@ -176,8 +179,8 @@ import { SubscriptionsStorage } from './types/subscriptions.js' export type { SubscriptionsStorage } import { UsageStorage } from './types/usage.js' export type { UsageStorage } -import { ReceiptsStorage } from './types/service.js' -export type { ReceiptsStorage } +import { ReceiptsStorage, TasksScheduler } from './types/service.js' +export type { ReceiptsStorage, TasksScheduler } import { AllocationsStorage, BlobsStorage, @@ -268,6 +271,11 @@ export interface Service extends StorefrontService, W3sService { } ucan: { + conclude: ServiceMethod< + UCANConclude, + UCANConcludeSuccess, + UCANConcludeFailure + > revoke: ServiceMethod } @@ -324,6 +332,7 @@ export type BlobServiceContext = SpaceServiceContext & { allocationsStorage: AllocationsStorage blobsStorage: BlobsStorage tasksStorage: TasksStorage + receiptsStorage: ReceiptsStorage getServiceConnection: () => ConnectionView } @@ -409,10 +418,22 @@ export interface RevocationServiceContext { } export interface ConcludeServiceContext { + /** + * Service signer + */ + id: Signer /** * Stores receipts for tasks. */ receiptsStorage: ReceiptsStorage + /** + * Stores tasks. + */ + tasksStorage: TasksStorage + /** + * Task scheduler. + */ + tasksScheduler: TasksScheduler } export interface PlanServiceContext { diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index 7a543b031..8adcd03f8 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -22,6 +22,10 @@ import { Storage } from './storage.js' export type TasksStorage = Storage export interface AllocationsStorage { + get: ( + space: DID, + blobMultihash: BlobMultihash + ) => Promise> exists: ( space: DID, blobMultihash: BlobMultihash @@ -55,6 +59,11 @@ export interface BlobAddInput { export interface BlobAddOutput extends Omit {} +export interface BlobGetOutput { + blob: { content: Uint8Array; size: number } + invocation: UnknownLink +} + export interface BlobsStorage { has: (content: BlobMultihash) => Promise> createUploadUrl: ( diff --git a/packages/upload-api/src/types/service.ts b/packages/upload-api/src/types/service.ts index 066d50438..6e9858572 100644 --- a/packages/upload-api/src/types/service.ts +++ b/packages/upload-api/src/types/service.ts @@ -1,4 +1,14 @@ -import type { UnknownLink, Receipt } from '@ucanto/interface' +import type { + UnknownLink, + Receipt, + Invocation, + Result, + Unit, + Failure, +} from '@ucanto/interface' import { Storage } from './storage.js' export type ReceiptsStorage = Storage +export interface TasksScheduler { + schedule: (invocation: Invocation) => Promise> +} diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index e88ba6f18..ead5c752b 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -1,6 +1,7 @@ import { provide } from '@ucanto/server' import { Message } from '@ucanto/core' import { CAR } from '@ucanto/transport' +import * as Blob from '@web3-storage/capabilities/blob' import { conclude } from '@web3-storage/capabilities/ucan' import * as API from '../types.js' @@ -8,10 +9,18 @@ import * as API from '../types.js' * @param {API.ConcludeServiceContext} context * @returns {API.ServiceMethod} */ -export const ucanConcludeProvider = ({ receiptsStorage }) => +export const ucanConcludeProvider = ({ + id, + receiptsStorage, + tasksStorage, + tasksScheduler, +}) => provide(conclude, async ({ capability }) => { const messageCar = CAR.codec.decode(capability.nb.bytes) - const message = Message.view({ root: messageCar.roots[0].cid, store: messageCar.blocks }) + const message = Message.view({ + root: messageCar.roots[0].cid, + store: messageCar.blocks, + }) // TODO: check number of receipts const receiptKey = Array.from(message.receipts.keys())[0] @@ -20,15 +29,52 @@ export const ucanConcludeProvider = ({ receiptsStorage }) => if (!receipt) { throw new Error('receipt should exist') } - + + // Store receipt const receiptPutRes = await receiptsStorage.put(receipt) if (receiptPutRes.error) { return { - error: receiptPutRes.error + error: receiptPutRes.error, } } - // TODO: Schedule accept (temporary simple hack) + // THIS IS A TEMPORARY HACK + // Schedule `blob/accept` + const ranInvocation = receipt.ran + + // TODO: This actually needs the accept task!!!! + // Get invocation + const httpPutTaskGetRes = await tasksStorage.get(ranInvocation.link()) + if (httpPutTaskGetRes.error) { + return httpPutTaskGetRes + } + + // Schedule `blob/accept` if there is a `http/put` capability + const scheduleRes = await Promise.all( + httpPutTaskGetRes.ok.capabilities + .filter((cap) => cap.can === Blob.put.can) + .map(async (cap) => { + const blobAccept = await Blob.accept.invoke({ + issuer: id, + audience: id, + with: id.toDIDKey(), + nb: { + // @ts-expect-error blob exists in put + blob: cap.nb.blob, + exp: Number.MAX_SAFE_INTEGER, + }, + expiration: Infinity, + }).delegate() + + return tasksScheduler.schedule(blobAccept) + }) + ) + const scheduleErrors = scheduleRes.filter(res => res.error) + if (scheduleErrors.length && scheduleErrors[0].error) { + return { + error: scheduleErrors[0].error + } + } return { ok: { time: Date.now() }, diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index a6d51afa6..0b8bd3c86 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -1,8 +1,10 @@ import * as API from '../../src/types.js' +import { equals } from 'uint8arrays' +import pDefer from 'p-defer' import { Absentee } from '@ucanto/principal' -import { Message } from '@ucanto/core' +import { Message, Receipt } from '@ucanto/core' +import { ed25519 } from '@ucanto/principal' import { CAR } from '@ucanto/transport' -import { equals } from 'uint8arrays' import { sha256 } from 'multiformats/hashes/sha2' import * as BlobCapabilities from '@web3-storage/capabilities/blob' import * as UCAN from '@web3-storage/capabilities/ucan' @@ -17,7 +19,7 @@ import { BlobItemSizeExceededName } from '../../src/blob/lib.js' * @type {API.Tests} */ export const test = { - 'blob/add executes allocation and returns effects for allocation, accept, put and allocate receipt': + 'blob/add executes allocation and returns effects for allocate (and its receipt), accept and put': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -53,7 +55,9 @@ export const test = { // Validate receipt assert.ok(blobAdd.out.ok.claim) - assert.ok(blobAdd.out.ok.claim['await/ok'].equals(blobAdd.fx.join?.link())) + assert.ok( + blobAdd.out.ok.claim['await/ok'].equals(blobAdd.fx.join?.link()) + ) assert.ok(blobAdd.fx.join) /** @@ -62,23 +66,38 @@ export const test = { // @ts-expect-error read only effect const forkInvocations = blobAdd.fx.fork assert.equal(blobAdd.fx.fork.length, 3) - const allocatefx = forkInvocations.find(fork => fork.capabilities[0].can === BlobCapabilities.allocate.can) - const allocateUcanConcludefx = forkInvocations.find(fork => fork.capabilities[0].can === UCAN.conclude.can) - const putfx = forkInvocations.find(fork => fork.capabilities[0].can === BlobCapabilities.put.can) + const allocatefx = forkInvocations.find( + (fork) => fork.capabilities[0].can === BlobCapabilities.allocate.can + ) + const allocateUcanConcludefx = forkInvocations.find( + (fork) => fork.capabilities[0].can === UCAN.conclude.can + ) + const putfx = forkInvocations.find( + (fork) => fork.capabilities[0].can === BlobCapabilities.put.can + ) if (!allocatefx || !allocateUcanConcludefx || !putfx) { throw new Error('effects not provided') } + // validate facts exist for `http/put` + assert.ok(putfx.facts.length) + const [{ bytes, did }] = putfx.facts + assert.ok(bytes) + assert.ok(did) + // Validate `http/put` invocation stored - const httpPutGetTask = await context.tasksStorage.get( - putfx.cid - ) + const httpPutGetTask = await context.tasksStorage.get(putfx.cid) assert.ok(httpPutGetTask.ok) // validate scheduled allocate task ran an its receipt content - // @ts-expect-error object of type unknown - const messageCar = CAR.codec.decode(allocateUcanConcludefx.capabilities[0].nb.bytes) - const message = Message.view({ root: messageCar.roots[0].cid, store: messageCar.blocks }) + const messageCar = CAR.codec.decode( + // @ts-expect-error object of type unknown + allocateUcanConcludefx.capabilities[0].nb.bytes + ) + const message = Message.view({ + root: messageCar.roots[0].cid, + store: messageCar.blocks, + }) const receiptKey = Array.from(message.receipts.keys())[0] const receipt = message.receipts.get(receiptKey) @@ -89,6 +108,122 @@ export const test = { // @ts-expect-error receipt out is unknown assert.ok(receipt?.out.ok?.address) }, + 'blob/add executes allocation and returns effects for allocate (and its receipt) and accept, but not for put when blob stored': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const invocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, + }, + proofs: [proof], + }) + // Invoke `blob/add` for the first time + const firstBlobAdd = await invocation.execute(connection) + if (!firstBlobAdd.out.ok) { + throw new Error('invocation failed', { cause: firstBlobAdd }) + } + + // Validate receipt + assert.ok(firstBlobAdd.out.ok) + assert.ok(firstBlobAdd.fx.join) + assert.equal(firstBlobAdd.fx.fork.length, 3) + + // Store allocate receipt + /** + * @type {import('@ucanto/interface').Invocation[]} + **/ + // @ts-expect-error read only effect + const forkInvocations = firstBlobAdd.fx.fork + const allocateUcanConcludefx = forkInvocations.find( + (fork) => fork.capabilities[0].can === UCAN.conclude.can + ) + const putfx = forkInvocations.find( + (fork) => fork.capabilities[0].can === BlobCapabilities.put.can + ) + if (!allocateUcanConcludefx || !putfx) { + throw new Error('effects not provided') + } + const messageCar = CAR.codec.decode( + // @ts-expect-error object of type unknown + allocateUcanConcludefx.capabilities[0].nb.bytes + ) + const message = Message.view({ + root: messageCar.roots[0].cid, + store: messageCar.blocks, + }) + const receiptKey = Array.from(message.receipts.keys())[0] + const receipt = message.receipts.get(receiptKey) + if (!receipt) { + throw new Error('receipt should be available') + } + const receiptPutRes = await context.receiptsStorage.put(receipt) + assert.ok(receiptPutRes.ok) + + // Invoke `blob/add` for the second time (without storing the blob) + const secondBlobAdd = await invocation.execute(connection) + if (!secondBlobAdd.out.ok) { + throw new Error('invocation failed', { cause: secondBlobAdd }) + } + + // Validate receipt has still 3 effects + assert.ok(secondBlobAdd.out.ok) + assert.ok(secondBlobAdd.fx.join) + assert.equal(secondBlobAdd.fx.fork.length, 3) + + /** @type {import('@web3-storage/capabilities/types').BlobAddress} */ + // @ts-expect-error receipt type is unknown + const address = receipt.out.ok.address + + // Store the blob to the address + const goodPut = await fetch(address.url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: address.headers, + }) + assert.equal(goodPut.status, 200, await goodPut.text()) + + // Invoke `blob/add` for the third time (after storing the blob) + const thirdBlobAdd = await invocation.execute(connection) + if (!thirdBlobAdd.out.ok) { + throw new Error('invocation failed', { cause: thirdBlobAdd }) + } + + // Validate receipt has now only 2 effects + assert.ok(thirdBlobAdd.out.ok) + assert.ok(thirdBlobAdd.fx.join) + assert.equal(thirdBlobAdd.fx.fork.length, 2) + + /** + * @type {import('@ucanto/interface').Invocation[]} + **/ + // @ts-expect-error read only effect + const thirdForkInvocations = thirdBlobAdd.fx.fork + // no put effect anymore + assert.ok(!thirdForkInvocations.find( + (fork) => fork.capabilities[0].can === BlobCapabilities.put.can + )) + }, 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -626,6 +761,149 @@ export const test = { const retryBlobAllocate = await serviceBlobAllocate.execute(connection) assert.equal(retryBlobAllocate.out.error, undefined) }, + 'blob/accept is executed once ucan/conclude is invoked with the blob put receipt and blob was sent': + async (assert, context) => { + const taskScheduled = pDefer() + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const content = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer({ + ...context, + tasksScheduler: { + schedule: (invocation) => { + taskScheduled.resolve(invocation) + + return Promise.resolve({ + ok: {}, + }) + } + }, + }), + }) + + // invoke `blob/add` + const invocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + content, + size, + }, + }, + proofs: [proof], + }) + const blobAdd = await invocation.execute(connection) + if (!blobAdd.out.ok) { + throw new Error('invocation failed', { cause: blobAdd }) + } + + // Get receipt relevant content + /** + * @type {import('@ucanto/interface').Invocation[]} + **/ + // @ts-expect-error read only effect + const forkInvocations = blobAdd.fx.fork + const allocateUcanConcludefx = forkInvocations.find( + (fork) => fork.capabilities[0].can === UCAN.conclude.can + ) + const putfx = forkInvocations.find( + (fork) => fork.capabilities[0].can === BlobCapabilities.put.can + ) + if (!allocateUcanConcludefx || !putfx) { + throw new Error('effects not provided') + } + const blobAllocateMessageCar = CAR.codec.decode( + // @ts-expect-error object of type unknown + allocateUcanConcludefx.capabilities[0].nb.bytes + ) + const blobAllocateMessage = Message.view({ + root: blobAllocateMessageCar.roots[0].cid, + store: blobAllocateMessageCar.blocks, + }) + const receiptKey = Array.from(blobAllocateMessage.receipts.keys())[0] + const receipt = blobAllocateMessage.receipts.get(receiptKey) + + // Get `blob/allocate` receipt with address + /** + * @type {import('@web3-storage/capabilities/types').BlobAddress} + **/ + // @ts-expect-error receipt out is unknown + const address = receipt?.out.ok?.address + assert.ok(address) + + // Write blob + const goodPut = await fetch(address.url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: address?.headers, + }) + assert.equal(goodPut.status, 200, await goodPut.text()) + + // Create `http/put` receipt + /** @type {{ bytes: Uint8Array, did: string }} */ + // @ts-expect-error facts are unknown + const [{ bytes, did }] = putfx.facts + const putSubject = ed25519.decode(bytes) + const httpPut = BlobCapabilities.put.invoke({ + issuer: putSubject, + audience: putSubject, + with: putSubject.toDIDKey(), + nb: { + blob: { + content, + size, + }, + address + }, + facts: putfx.facts, + expiration: Infinity, + }) + + const httpPutDelegation = await httpPut.delegate() + const httpPutReceipt = await Receipt.issue({ + issuer: putSubject, + ran: httpPutDelegation.cid, + result: { + ok: {}, + }, + }) + const message = await Message.build({ receipts: [httpPutReceipt] }) + const messageCar = await CAR.outbound.encode(message) + + // Invoke `ucan/conclude` with `http/put` receipt + const httpPutConcludeInvocation = UCAN.conclude.invoke({ + issuer: alice, + audience: context.id, + with: alice.did(), + nb: { + bytes: messageCar.body, + }, + expiration: Infinity, + }) + const ucanConclude = await httpPutConcludeInvocation.execute(connection) + if (!ucanConclude.out.ok) { + throw new Error('invocation failed', { cause: blobAdd }) + } + + // verify accept was scheduled + const blobAcceptInvocation = await taskScheduled.promise + assert.ok(blobAcceptInvocation) + assert.ok( + blobAdd.out.ok.claim['await/ok'].equals(blobAcceptInvocation.cid) + ) + assert.ok(blobAdd.fx.join?.link().equals(blobAcceptInvocation.cid)) + }, // TODO: Blob accept // TODO: list // TODO: remove diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index 1926fb63a..e80fdd53b 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -59,6 +59,12 @@ export const createContext = async ( requirePaymentPlan, url: new URL('http://localhost:8787'), ...serviceStores, + tasksScheduler: { + schedule: () => + Promise.resolve({ + ok: {}, + }), + }, getServiceConnection: () => connection, ...createRevocationChecker({ revocationsStorage: serviceStores.revocationsStorage, diff --git a/packages/upload-api/test/storage/allocations-storage.js b/packages/upload-api/test/storage/allocations-storage.js index d6982bd59..67946a97b 100644 --- a/packages/upload-api/test/storage/allocations-storage.js +++ b/packages/upload-api/test/storage/allocations-storage.js @@ -33,6 +33,21 @@ export class AllocationsStorage { return { ok: output } } + /** + * @param {Types.DID} space + * @param {Uint8Array} blobMultihash + * @returns {ReturnType} + */ + async get(space, blobMultihash) { + const item = this.items.find( + (i) => i.space === space && equals(i.blob.content, blobMultihash) + ) + if (!item) { + return { error: { name: 'RecordNotFound', message: 'record not found' } } + } + return { ok: item } + } + /** * @param {Types.DID} space * @param {Uint8Array} blobMultihash diff --git a/packages/upload-api/test/storage/receipts-storage.js b/packages/upload-api/test/storage/receipts-storage.js index 91f876e0b..914fef611 100644 --- a/packages/upload-api/test/storage/receipts-storage.js +++ b/packages/upload-api/test/storage/receipts-storage.js @@ -14,7 +14,7 @@ import { RecordNotFound } from '../../src/errors.js' */ export class ReceiptsStorage { constructor() { - /** @type {Map} */ + /** @type {Map} */ this.items = new Map() } @@ -23,7 +23,7 @@ export class ReceiptsStorage { * @returns {Promise>} */ async put(record) { - this.items.set(record.ran.link(), record) + this.items.set(record.ran.link().toString(), record) return Promise.resolve({ ok: {}, @@ -35,7 +35,7 @@ export class ReceiptsStorage { * @returns {Promise>} */ async get(link) { - const record = this.items.get(link) + const record = this.items.get(link.toString()) if (!record) { return { error: new RecordNotFound('not found'), @@ -51,7 +51,7 @@ export class ReceiptsStorage { * @returns {Promise>} */ async has(link) { - const record = this.items.get(link) + const record = this.items.get(link.toString()) if (!record) { return { ok: false, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5fdd31cc8..566bb02aa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -452,6 +452,9 @@ importers: one-webcrypto: specifier: git://github.com/web3-storage/one-webcrypto version: github.com/web3-storage/one-webcrypto/5148cd14d5489a8ac4cd38223870e02db15a2382 + p-defer: + specifier: ^4.0.1 + version: 4.0.1 typescript: specifier: 5.2.2 version: 5.2.2 @@ -8393,7 +8396,7 @@ packages: /it-parallel@3.0.6: resolution: {integrity: sha512-i7UM7I9LTkDJw3YIqXHFAPZX6CWYzGc+X3irdNrVExI4vPazrJdI7t5OqrSVN8CONXLAunCiqaSV/zZRbQR56A==} dependencies: - p-defer: 4.0.0 + p-defer: 4.0.1 dev: true /it-pipe@2.0.5: @@ -8408,7 +8411,7 @@ packages: /it-pushable@3.2.3: resolution: {integrity: sha512-gzYnXYK8Y5t5b/BnJUr7glfQLO4U5vyb05gPx/TyTw+4Bv1zM9gFk4YsOrnulWefMewlphCjKkakFvj1y99Tcg==} dependencies: - p-defer: 4.0.0 + p-defer: 4.0.1 dev: true /it-stream-types@1.0.5: @@ -9976,6 +9979,12 @@ packages: /p-defer@4.0.0: resolution: {integrity: sha512-Vb3QRvQ0Y5XnF40ZUWW7JfLogicVh/EnA5gBIvKDJoYpeI82+1E3AlB9yOcKFS0AhHrWVnAQO39fbR0G99IVEQ==} engines: {node: '>=12'} + dev: false + + /p-defer@4.0.1: + resolution: {integrity: sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==} + engines: {node: '>=12'} + dev: true /p-event@6.0.0: resolution: {integrity: sha512-Xbfxd0CfZmHLGKXH32k1JKjQYX6Rkv0UtQdaFJ8OyNcf+c0oWCeXHc1C4CX/IESZLmcvfPa5aFIO/vCr5gqtag==} From 05bade819fa936ceb8d70fdf750db1c0c859d2f8 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 5 Apr 2024 15:25:07 +0200 Subject: [PATCH 05/27] chore: address review --- packages/capabilities/package.json | 4 + packages/capabilities/src/blob.js | 200 +----------------- packages/capabilities/src/http.js | 55 +++++ packages/capabilities/src/index.js | 12 +- packages/capabilities/src/types.ts | 46 ++-- packages/capabilities/src/ucan.js | 43 +--- .../capabilities/src/web3.storage/blob.js | 102 +++++++++ packages/upload-api/src/blob.js | 4 - packages/upload-api/src/blob/accept.js | 5 +- packages/upload-api/src/blob/add.js | 102 +++++---- packages/upload-api/src/blob/allocate.js | 150 ++++++------- packages/upload-api/src/blob/lib.js | 8 +- packages/upload-api/src/blob/list.js | 15 -- packages/upload-api/src/blob/remove.js | 22 -- packages/upload-api/src/errors.js | 13 ++ packages/upload-api/src/types.ts | 8 - packages/upload-api/src/ucan/conclude.js | 72 +++++-- packages/upload-api/test/handlers/blob.js | 105 +++++---- 18 files changed, 476 insertions(+), 490 deletions(-) create mode 100644 packages/capabilities/src/http.js create mode 100644 packages/capabilities/src/web3.storage/blob.js delete mode 100644 packages/upload-api/src/blob/list.js delete mode 100644 packages/upload-api/src/blob/remove.js diff --git a/packages/capabilities/package.json b/packages/capabilities/package.json index 34bc7711e..ecf27a2f7 100644 --- a/packages/capabilities/package.json +++ b/packages/capabilities/package.json @@ -56,6 +56,10 @@ "types": "./dist/src/filecoin/dealer.d.ts", "import": "./src/filecoin/dealer.js" }, + "./web3.storage/blob": { + "types": "./dist/src/web3.storage/blob.d.ts", + "import": "./src/web3.storage/blob.js" + }, "./types": { "types": "./dist/src/types.d.ts", "import": "./src/types.js" diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 163781126..a9108dcd4 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -11,16 +11,8 @@ * * @module */ -import { capability, Link, Schema, ok, fail } from '@ucanto/validator' -import { - equal, - equalBlob, - equalContent, - equalWith, - checkLink, - SpaceDID, - and, -} from './utils.js' +import { capability, Schema } from '@ucanto/validator' +import { equalBlob, equalWith, SpaceDID } from './utils.js' /** * Agent capabilities for Blob protocol @@ -79,192 +71,6 @@ export const add = capability({ derives: equalBlob, }) -/** - * `blob/remove` capability can be used to remove the stored Blob from the (memory) - * space identified by `with` field. - */ -export const remove = capability({ - can: 'blob/remove', - /** - * DID of the (memory) space where Blob is intended to - * be stored. - */ - with: SpaceDID, - nb: Schema.struct({ - /** - * A multihash digest of the blob payload bytes, uniquely identifying blob. - */ - content: Schema.bytes(), - }), - derives: equalContent, -}) - -/** - * `blob/list` capability can be invoked to request a list of stored Blobs in the - * (memory) space identified by `with` field. - */ -export const list = capability({ - can: 'blob/list', - /** - * DID of the (memory) space where Blob is intended to - * be stored. - */ - with: SpaceDID, - nb: Schema.struct({ - /** - * A pointer that can be moved back and forth on the list. - * It can be used to paginate a list for instance. - */ - cursor: Schema.string().optional(), - /** - * Maximum number of items per page. - */ - size: Schema.integer().optional(), - /** - * If true, return page of results preceding cursor. Defaults to false. - */ - pre: Schema.boolean().optional(), - }), - derives: (claimed, delegated) => { - if (claimed.with !== delegated.with) { - return fail( - `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` - ) - } - return ok({}) - }, -}) - -/** - * Service capabilities for Blob protocol - */ -/** - * Capability can only be delegated (but not invoked) allowing audience to - * derived any `web3.storage/blob/` prefixed capability for the (memory) space identified - * by DID in the `with` field. - */ -export const serviceBlob = capability({ - can: 'web3.storage/blob/*', - /** - * DID of the (memory) space where Blob is intended to - * be stored. - */ - with: SpaceDID, - derives: equalWith, -}) - -/** - * `web3.storage/blob//allocate` capability can be invoked to create a memory - * address where blob content can be written via HTTP PUT request. - */ -export const allocate = capability({ - can: 'web3.storage/blob/allocate', - /** - * Provider DID. - */ - with: Schema.did(), - nb: Schema.struct({ - /** - * Blob to allocate on the space. - */ - blob: blobStruct, - /** - * The Link for an Add Blob task, that caused an allocation - */ - cause: Link, - /** - * DID of the user space where allocation takes place - */ - space: SpaceDID, - }), - derives: (claim, from) => { - return ( - and(equalWith(claim, from)) || - and(equalBlob(claim, from)) || - and(checkLink(claim.nb.cause, from.nb.cause, 'cause')) || - and(equal(claim.nb.space, from.nb.space, 'space')) || - ok({}) - ) - }, -}) - -/** - * `http/put` capability invocation MAY be performed by any agent on behalf of the subject. - * The `blob/add` provider MUST add `/http/put` effect and capture private key of the - * `subject` in the `meta` field so that any agent could perform it. - */ -export const put = capability({ - can: 'http/put', - /** - * DID of the (memory) space where Blob is intended to - * be stored. - */ - with: SpaceDID, - nb: Schema.struct({ - /** - * Blob to allocate on the space. - */ - blob: blobStruct, - /** - * Blob to accept. - */ - address: Schema.struct({ - /** - * HTTP(S) location that can receive blob content via HTTP PUT request. - */ - url: Schema.string(), - /** - * HTTP headers. - */ - headers: Schema.unknown(), - }).optional(), - }), - derives: (claim, from) => { - return ( - and(equalWith(claim, from)) || - and(equalBlob(claim, from)) || - and(equal(claim.nb.address?.url, from.nb.address, 'url')) || - and(equal(claim.nb.address?.headers, from.nb.address, 'headers')) || - ok({}) - ) - }, -}) - -/** - * `blob/accept` capability invocation should either succeed when content is - * delivered on allocated address or fail if no content is allocation expires - * without content being delivered. - */ -export const accept = capability({ - can: 'web3.storage/blob/accept', - /** - * Provider DID. - */ - with: Schema.did(), - nb: Schema.struct({ - /** - * Blob to accept. - */ - blob: blobStruct, - /** - * Expiration.. - */ - exp: Schema.integer(), - }), - derives: (claim, from) => { - const result = equalBlob(claim, from) - if (result.error) { - return result - } else if (claim.nb.exp !== undefined && from.nb.exp !== undefined) { - return claim.nb.exp > from.nb.exp - ? fail(`exp constraint violation: ${claim.nb.exp} > ${from.nb.exp}`) - : ok({}) - } else { - return ok({}) - } - }, -}) - // ⚠️ We export imports here so they are not omitted in generated typedes // @see https://github.com/microsoft/TypeScript/issues/51548 -export { Schema, Link } +export { Schema } diff --git a/packages/capabilities/src/http.js b/packages/capabilities/src/http.js new file mode 100644 index 000000000..cf171ee4c --- /dev/null +++ b/packages/capabilities/src/http.js @@ -0,0 +1,55 @@ +/** + * HTTP Capabilities. + * + * These can be imported directly with: + * ```js + * import * as HTTP from '@web3-storage/capabilities/http' + * ``` + * + * @module + */ +import { capability, Schema, ok } from '@ucanto/validator' +import { blobStruct } from './blob.js' +import { equal, equalBlob, equalWith, SpaceDID, and } from './utils.js' + +/** + * `http/put` capability invocation MAY be performed by any agent on behalf of the subject. + * The `blob/add` provider MUST add `/http/put` effect and capture private key of the + * `subject` in the `meta` field so that any agent could perform it. + */ +export const put = capability({ + can: 'http/put', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * Blob to allocate on the space. + */ + blob: blobStruct, + /** + * Blob to accept. + */ + address: Schema.struct({ + /** + * HTTP(S) location that can receive blob content via HTTP PUT request. + */ + url: Schema.string(), + /** + * HTTP headers. + */ + headers: Schema.unknown(), + }).optional(), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(equalBlob(claim, from)) || + and(equal(claim.nb.address?.url, from.nb.address, 'url')) || + and(equal(claim.nb.address?.headers, from.nb.address, 'headers')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index fc7e4bc7c..c5d9385d1 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -20,6 +20,8 @@ import * as UCAN from './ucan.js' import * as Plan from './plan.js' import * as Usage from './usage.js' import * as Blob from './blob.js' +import * as W3sBlob from './web3.storage/blob.js' +import * as HTTP from './http.js' export { Access, @@ -90,10 +92,8 @@ export const abilitiesAsStrings = [ Usage.report.can, Blob.blob.can, Blob.add.can, - Blob.remove.can, - Blob.list.can, - Blob.serviceBlob.can, - Blob.put.can, - Blob.allocate.can, - Blob.accept.can, + W3sBlob.blob.can, + W3sBlob.allocate.can, + W3sBlob.accept.can, + HTTP.put.can, ] diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 64c99825c..2177b9958 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -22,6 +22,8 @@ import { space, info } from './space.js' import * as provider from './provider.js' import { top } from './top.js' import * as BlobCaps from './blob.js' +import * as W3sBlobCaps from './web3.storage/blob.js' +import * as HTTPCaps from './http.js' import * as StoreCaps from './store.js' import * as UploadCaps from './upload.js' import * as AccessCaps from './access.js' @@ -440,29 +442,33 @@ export interface UploadNotFound extends Ucanto.Failure { export type UploadGetFailure = UploadNotFound | Ucanto.Failure +// HTTP +export type HTTPPut = InferInvokedCapability + // Blob export type Blob = InferInvokedCapability export type BlobAdd = InferInvokedCapability -export type BlobRemove = InferInvokedCapability -export type BlobList = InferInvokedCapability -export type ServiceBlob = InferInvokedCapability -export type BlobPut = InferInvokedCapability -export type BlobAllocate = InferInvokedCapability -export type BlobAccept = InferInvokedCapability +export type ServiceBlob = InferInvokedCapability +export type BlobAllocate = InferInvokedCapability +export type BlobAccept = InferInvokedCapability export type BlobMultihash = Uint8Array +export interface BlobModel { + content: BlobMultihash + size: number +} // Blob add export interface BlobAddSuccess { - claim: { - 'await/ok': Link + location: { + 'ucan/await': ['.out.ok.claim', Link] } } -export interface BlobItemSizeExceeded extends Ucanto.Failure { - name: 'BlobItemSizeExceeded' +export interface BlobExceedsSizeLimit extends Ucanto.Failure { + name: 'BlobExceedsSizeLimit' } -export type BlobAddFailure = BlobItemSizeExceeded | Ucanto.Failure +export type BlobAddFailure = BlobExceedsSizeLimit | Ucanto.Failure // Blob remove export interface BlobRemoveSuccess { @@ -473,12 +479,13 @@ export interface BlobItemNotFound extends Ucanto.Failure { name: 'BlobItemNotFound' } +// TODO: Add more errors from stores export type BlobRemoveFailure = BlobItemNotFound | Ucanto.Failure // Blob list export interface BlobListSuccess extends ListResponse {} export interface BlobListItem { - blob: { content: Uint8Array; size: number } + blob: BlobModel insertedAt: ISO8601Date } @@ -657,11 +664,14 @@ export type UCANRevokeFailure = | UnauthorizedRevocation | RevocationsStoreFailure -export interface InvocationNotFound extends Ucanto.Failure { - name: 'InvocationNotFound' +/** + * Error is raised when receipt is received for unknown invocation + */ +export interface InvocationNotFoundForReceipt extends Ucanto.Failure { + name: 'InvocationNotFoundForReceipt' } -export type UCANConcludeFailure = InvocationNotFound | Ucanto.Failure +export type UCANConcludeFailure = InvocationNotFoundForReceipt | Ucanto.Failure // Admin export type Admin = InferInvokedCapability @@ -797,12 +807,10 @@ export type ServiceAbilityArray = [ UsageReport['can'], Blob['can'], BlobAdd['can'], - BlobRemove['can'], - BlobList['can'], ServiceBlob['can'], - BlobPut['can'], BlobAllocate['can'], - BlobAccept['can'] + BlobAccept['can'], + HTTPPut['can'] ] /** diff --git a/packages/capabilities/src/ucan.js b/packages/capabilities/src/ucan.js index c703f2f97..51a07ffe0 100644 --- a/packages/capabilities/src/ucan.js +++ b/packages/capabilities/src/ucan.js @@ -2,7 +2,7 @@ * UCAN core capabilities. */ -import { capability, Schema } from '@ucanto/validator' +import { capability, Schema, ok } from '@ucanto/validator' import * as API from '@ucanto/interface' import { equalWith, equal, and, checkLink } from './utils.js' @@ -89,44 +89,17 @@ export const conclude = capability({ * MUST be the DID of the audience of the ran invocation. */ with: Schema.did(), - // TODO: Should this just have bytes? nb: Schema.struct({ - bytes: Schema.Bytes, - // /** - // * A link to the UCAN invocation that this receipt is for. - // */ - // ran: UCANLink, - // /** - // * The value output of the invocation in Result format. - // */ - // out: Schema.unknown(), - // /** - // * Tasks that the invocation would like to enqueue. - // */ - // next: Schema.array(UCANLink), - // /** - // * Additional data about the receipt - // */ - // meta: Schema.unknown(), - // /** - // * The UTC Unix timestamp at which the Receipt was issued - // */ - // time: Schema.integer(), + /** + * CID of the content with the UCANTO Message. + */ + message: Schema.link(), }), derives: (claim, from) => // With field MUST be the same - and(equalWith(claim, from)) ?? - equal(claim.nb.bytes, from.nb.bytes, 'nb.bytes'), - // // invocation MUST be the same - // and(checkLink(claim.nb.ran, from.nb.ran, 'nb.ran')) ?? - // // value output MUST be the same - // and(equal(claim.nb.out, from.nb.out, 'nb.out')) ?? - // // tasks to enqueue MUST be the same - // and(equal(claim.nb.next, from.nb.next, 'nb.next')) ?? - // // additional data MUST be the same - // and(equal(claim.nb.meta, from.nb.meta, 'nb.meta')) ?? - // // the receipt issue time MUST be the same - // equal(claim.nb.time, from.nb.time, 'nb.time'), + and(equalWith(claim, from)) || + and(checkLink(claim.nb.message, from.nb.message, 'nb.message')) || + ok({}), }) /** diff --git a/packages/capabilities/src/web3.storage/blob.js b/packages/capabilities/src/web3.storage/blob.js new file mode 100644 index 000000000..56188c847 --- /dev/null +++ b/packages/capabilities/src/web3.storage/blob.js @@ -0,0 +1,102 @@ +import { capability, Schema, Link, ok, fail } from '@ucanto/validator' +import { blobStruct } from '../blob.js' +import { + equalBlob, + equalWith, + SpaceDID, + and, + equal, + checkLink, +} from '../utils.js' + +/** + * Service capabilities for Blob protocol + */ +/** + * Capability can only be delegated (but not invoked) allowing audience to + * derived any `web3.storage/blob/` prefixed capability for the (memory) space identified + * by DID in the `with` field. + */ +export const blob = capability({ + can: 'web3.storage/blob/*', + /** + * DID of the (memory) space where Blob is intended to + * be stored. + */ + with: SpaceDID, + derives: equalWith, +}) + +/** + * `web3.storage/blob//allocate` capability can be invoked to create a memory + * address where blob content can be written via HTTP PUT request. + */ +export const allocate = capability({ + can: 'web3.storage/blob/allocate', + /** + * Provider DID. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * Blob to allocate on the space. + */ + blob: blobStruct, + /** + * The Link for an Add Blob task, that caused an allocation + */ + cause: Link, + /** + * DID of the user space where allocation takes place + */ + space: SpaceDID, + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(equalBlob(claim, from)) || + and(checkLink(claim.nb.cause, from.nb.cause, 'cause')) || + and(equal(claim.nb.space, from.nb.space, 'space')) || + ok({}) + ) + }, +}) + +/** + * `blob/accept` capability invocation should either succeed when content is + * delivered on allocated address or fail if no content is allocation expires + * without content being delivered. + */ +export const accept = capability({ + can: 'web3.storage/blob/accept', + /** + * Provider DID. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * Blob to accept. + */ + blob: blobStruct, + /** + * Expiration.. + */ + exp: Schema.integer(), + }), + derives: (claim, from) => { + const result = equalBlob(claim, from) + if (result.error) { + return result + } else if (claim.nb.exp !== undefined && from.nb.exp !== undefined) { + return claim.nb.exp > from.nb.exp + ? fail(`exp constraint violation: ${claim.nb.exp} > ${from.nb.exp}`) + : ok({}) + } else { + return ok({}) + } + }, +}) + +// ⚠️ We export imports here so they are not omitted in generated typedes +// @see https://github.com/microsoft/TypeScript/issues/51548 +export { Schema, Link } diff --git a/packages/upload-api/src/blob.js b/packages/upload-api/src/blob.js index 78b4bb40b..84187cefd 100644 --- a/packages/upload-api/src/blob.js +++ b/packages/upload-api/src/blob.js @@ -1,6 +1,4 @@ import { blobAddProvider } from './blob/add.js' -import { blobListProvider } from './blob/list.js' -import { blobRemoveProvider } from './blob/remove.js' import * as API from './types.js' /** @@ -9,7 +7,5 @@ import * as API from './types.js' export function createService(context) { return { add: blobAddProvider(context), - list: blobListProvider(context), - remove: blobRemoveProvider(context), } } diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index 07113aaa3..f7a26af44 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -1,5 +1,5 @@ import * as Server from '@ucanto/server' -import * as Blob from '@web3-storage/capabilities/blob' +import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import * as API from '../types.js' import { BlobItemNotFound } from './lib.js' @@ -8,7 +8,7 @@ import { BlobItemNotFound } from './lib.js' * @returns {API.ServiceMethod} */ export function blobAcceptProvider(context) { - return Server.provide(Blob.accept, async ({ capability }) => { + return Server.provide(W3sBlob.accept, async ({ capability }) => { const { blob } = capability.nb // If blob is not stored, we must fail const hasBlob = await context.blobsStorage.has(blob.content) @@ -18,6 +18,7 @@ export function blobAcceptProvider(context) { } } + // TODO: Set bucket name // TODO: return content commitment return { diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 86f28b5ec..300d097ea 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -3,10 +3,12 @@ import { Message } from '@ucanto/core' import { ed25519 } from '@ucanto/principal' import { CAR } from '@ucanto/transport' import * as Blob from '@web3-storage/capabilities/blob' +import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' +import * as HTTP from '@web3-storage/capabilities/http' import * as UCAN from '@web3-storage/capabilities/ucan' import * as API from '../types.js' -import { BlobItemSizeExceeded } from './lib.js' +import { BlobExceedsSizeLimit } from './lib.js' /** * @param {API.BlobServiceContext} context @@ -31,21 +33,23 @@ export function blobAddProvider(context) { // Verify blob is within accept size if (blob.size > maxUploadSize) { return { - error: new BlobItemSizeExceeded(maxUploadSize), + error: new BlobExceedsSizeLimit(maxUploadSize), } } - // Create shared subject for agent to issue `http/put` receipt + // We derive principal from the content multihash to be an audience + // of the `http/put` invocation. That way anyone with blob content + // could perform the invocation and issue receipt by deriving same + // principal const putSubject = await ed25519.derive(blob.content.slice(0, 32)) - const facts = Object.entries(putSubject.toArchive().keys).map( - ([key, value]) => ({ - did: key, - bytes: value, - }) - ) + const facts = [ + { + keys: putSubject.toArchive(), + }, + ] // Create web3.storage/blob/* invocations - const blobAllocate = Blob.allocate.invoke({ + const blobAllocate = W3sBlob.allocate.invoke({ issuer: id, audience: id, with: id.did(), @@ -56,7 +60,7 @@ export function blobAddProvider(context) { }, expiration: Infinity, }) - const blobAccept = Blob.accept.invoke({ + const blobAccept = W3sBlob.accept.invoke({ issuer: id, audience: id, with: id.toDIDKey(), @@ -71,12 +75,8 @@ export function blobAddProvider(context) { blobAccept.delegate(), ]) - // Get receipt for `blob/allocate` if available, or schedule invocation if not - const allocatedGetRes = await allocationsStorage.get( - space, - blob.content - ) + const allocatedGetRes = await allocationsStorage.get(space, blob.content) let blobAllocateReceipt let blobAllocateOutAddress // If already allocated, just get the allocate receipt @@ -114,37 +114,43 @@ export function blobAddProvider(context) { // Create `blob/allocate` receipt invocation to inline as effect const message = await Message.build({ receipts: [blobAllocateReceipt] }) const messageCar = await CAR.outbound.encode(message) + const bytes = new Uint8Array(messageCar.body) + const messageLink = await CAR.codec.link(bytes) + const allocateUcanConcludefx = await UCAN.conclude .invoke({ issuer: id, audience: id, with: id.toDIDKey(), nb: { - bytes: messageCar.body, + message: messageLink, }, expiration: Infinity, }) .delegate() + allocateUcanConcludefx.attach({ + bytes, + cid: messageLink, + }) // Create result object /** @type {API.OkBuilder} */ const result = Server.ok({ - claim: { - 'await/ok': acceptfx.link(), + location: { + 'ucan/await': ['.out.ok.claim', acceptfx.link()], }, }) // In case blob allocate provided an address to write // the blob is still not stored if (blobAllocateOutAddress) { - // Create effects for `blob/add` receipt - const blobPut = Blob.put.invoke({ + const blobPut = HTTP.put.invoke({ issuer: putSubject, audience: putSubject, with: putSubject.toDIDKey(), nb: { blob, - address: blobAllocateOutAddress + address: blobAllocateOutAddress, }, facts, expiration: Infinity, @@ -167,38 +173,44 @@ export function blobAddProvider(context) { } } - return result - // 1. System attempts to allocate memory in user space for the blob. - .fork(allocatefx) - .fork(allocateUcanConcludefx) - // 2. System requests user agent (or anyone really) to upload the content - // corresponding to the blob - // via HTTP PUT to given location. - .fork(putfx) - // 3. System will attempt to accept uploaded content that matches blob - // multihash and size. - .join(acceptfx) + return ( + result + // 1. System attempts to allocate memory in user space for the blob. + .fork(allocatefx) + .fork(allocateUcanConcludefx) + // 2. System requests user agent (or anyone really) to upload the content + // corresponding to the blob + // via HTTP PUT to given location. + .fork(putfx) + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. + .join(acceptfx) + ) } // Add allocate receipt if allocate was executed if (allocateUcanConcludefx) { - return result - // 1. System attempts to allocate memory in user space for the blob. + return ( + result + // 1. System attempts to allocate memory in user space for the blob. + .fork(allocatefx) + .fork(allocateUcanConcludefx) + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. + .join(acceptfx) + ) + } + + // Blob was already allocated and is already stored in the system + return ( + result + // 1. System allocated memory in user space for the blob. .fork(allocatefx) .fork(allocateUcanConcludefx) // 3. System will attempt to accept uploaded content that matches blob // multihash and size. .join(acceptfx) - } - - // Blob was already allocated and is already stored in the system - return result - // 1. System allocated memory in user space for the blob. - .fork(allocatefx) - .fork(allocateUcanConcludefx) - // 3. System will attempt to accept uploaded content that matches blob - // multihash and size. - .join(acceptfx) + ) }, }) } diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index f61cbfa69..bf78d5691 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -1,7 +1,6 @@ import * as Server from '@ucanto/server' -import * as Blob from '@web3-storage/capabilities/blob' +import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import * as API from '../types.js' -import { BlobItemNotFound } from './lib.js' import { ensureRateLimitAbove } from '../utils/rate-limits.js' /** @@ -9,92 +8,95 @@ import { ensureRateLimitAbove } from '../utils/rate-limits.js' * @returns {API.ServiceMethod} */ export function blobAllocateProvider(context) { - return Server.provide(Blob.allocate, async ({ capability, invocation }) => { - const { blob, cause, space } = capability.nb + return Server.provide( + W3sBlob.allocate, + async ({ capability, invocation }) => { + const { blob, cause, space } = capability.nb - // Rate limiting validation - const rateLimitResult = await ensureRateLimitAbove( - context.rateLimitsStorage, - [space], - 0 - ) - if (rateLimitResult.error) { - return { - error: { - name: 'InsufficientStorage', - message: `${space} is blocked`, - }, + // Rate limiting validation + // TODO: we should not produce rate limit error but rather suspend / queue task to be run after enforcing a limit without erroring + const rateLimitResult = await ensureRateLimitAbove( + context.rateLimitsStorage, + [space], + 0 + ) + if (rateLimitResult.error) { + return { + error: { + name: 'RateLimited', + message: `${space} is blocked`, + }, + } } - } - // Has Storage provider validation - const result = await context.provisionsStorage.hasStorageProvider(space) - if (result.error) { - return result - } - if (!result.ok) { - return { - /** @type {API.AllocationError} */ - error: { - name: 'InsufficientStorage', - message: `${space} has no storage provider`, - }, + // Has Storage provider validation + const result = await context.provisionsStorage.hasStorageProvider(space) + if (result.error) { + return result + } + if (!result.ok) { + return { + /** @type {API.AllocationError} */ + error: { + name: 'InsufficientStorage', + message: `${space} has no storage provider`, + }, + } } - } - // Check if blob already exists - const hasBlob = await context.blobsStorage.has(blob.content) - if (hasBlob.error) { - return { - error: new BlobItemNotFound(space), + // Allocate in space, ignoring if already allocated + const allocationInsert = await context.allocationsStorage.insert({ + space, + blob, + invocation: cause, + }) + if (allocationInsert.error) { + // if the insert failed with conflict then this item has already been + // added to the space and there is no allocation change. + if (allocationInsert.error.name === 'RecordKeyConflict') { + return { + ok: { size: 0 }, + } + } + return { + error: new Server.Failure('failed to allocate blob bytes'), + } } - } - // Get presigned URL for the write target - const createUploadUrl = await context.blobsStorage.createUploadUrl( - blob.content, - blob.size - ) - if (createUploadUrl.error) { - return { - error: new Server.Failure('failed to provide presigned url'), + + // Check if blob already exists + const hasBlobStore = await context.blobsStorage.has(blob.content) + if (hasBlobStore.error) { + return hasBlobStore } - } - const address = { - url: createUploadUrl.ok.url.toString(), - headers: createUploadUrl.ok.headers, - } - // Allocate in space, ignoring if already allocated - const allocationInsert = await context.allocationsStorage.insert({ - space, - blob, - invocation: cause, - }) - if (allocationInsert.error) { - // if the insert failed with conflict then this item has already been - // added to the space and there is no allocation change. - if (allocationInsert.error.name === 'RecordKeyConflict') { + // If blob is stored, we can just allocate it to the space + if (hasBlobStore.ok) { return { - ok: { size: 0 }, + ok: { size: blob.size }, } } - return { - error: new Server.Failure('failed to allocate blob bytes'), + + // Get presigned URL for the write target + const createUploadUrl = await context.blobsStorage.createUploadUrl( + blob.content, + blob.size + ) + if (createUploadUrl.error) { + return { + error: new Server.Failure('failed to provide presigned url'), + } + } + const address = { + url: createUploadUrl.ok.url.toString(), + headers: createUploadUrl.ok.headers, } - } - // If blob is stored, we can just allocate it to the space - if (hasBlob.ok) { return { - ok: { size: blob.size }, + ok: { + size: blob.size, + address, + }, } } - - return { - ok: { - size: blob.size, - address, - }, - } - }) + ) } diff --git a/packages/upload-api/src/blob/lib.js b/packages/upload-api/src/blob/lib.js index ff3995109..0e4b1b86a 100644 --- a/packages/upload-api/src/blob/lib.js +++ b/packages/upload-api/src/blob/lib.js @@ -29,8 +29,8 @@ export class BlobItemNotFound extends Failure { } } -export const BlobItemSizeExceededName = 'BlobItemSizeExceeded' -export class BlobItemSizeExceeded extends Failure { +export const BlobExceedsSizeLimitName = 'BlobExceedsSizeLimit' +export class BlobExceedsSizeLimit extends Failure { /** * @param {Number} maxUploadSize */ @@ -40,11 +40,11 @@ export class BlobItemSizeExceeded extends Failure { } get name() { - return BlobItemSizeExceededName + return BlobExceedsSizeLimitName } describe() { - return `Maximum size exceeded: ${this.maxUploadSize}, split DAG into smaller shards.` + return `Blob exceeded maximum size limit: ${this.maxUploadSize}, consider splitting it into blobs that fit limit.` } toJSON() { diff --git a/packages/upload-api/src/blob/list.js b/packages/upload-api/src/blob/list.js deleted file mode 100644 index c6c34bdb7..000000000 --- a/packages/upload-api/src/blob/list.js +++ /dev/null @@ -1,15 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Blob from '@web3-storage/capabilities/blob' -import * as API from '../types.js' - -/** - * @param {API.BlobServiceContext} context - * @returns {API.ServiceMethod} - */ -export function blobListProvider(context) { - return Server.provide(Blob.list, async ({ capability }) => { - const { cursor, size, pre } = capability.nb - const space = Server.DID.parse(capability.with).did() - return await context.allocationsStorage.list(space, { size, cursor, pre }) - }) -} diff --git a/packages/upload-api/src/blob/remove.js b/packages/upload-api/src/blob/remove.js deleted file mode 100644 index fb2e8c2d8..000000000 --- a/packages/upload-api/src/blob/remove.js +++ /dev/null @@ -1,22 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Blob from '@web3-storage/capabilities/blob' -import * as API from '../types.js' -import { BlobItemNotFound } from './lib.js' - -/** - * @param {API.BlobServiceContext} context - * @returns {API.ServiceMethod} - */ -export function blobRemoveProvider(context) { - return Server.provide(Blob.remove, async ({ capability }) => { - const { content } = capability.nb - const space = Server.DID.parse(capability.with).did() - - const res = await context.allocationsStorage.remove(space, content) - if (res.error && res.error.name === 'RecordNotFound') { - return Server.error(new BlobItemNotFound(space)) - } - - return res - }) -} diff --git a/packages/upload-api/src/errors.js b/packages/upload-api/src/errors.js index 13620c3a1..885c8a080 100644 --- a/packages/upload-api/src/errors.js +++ b/packages/upload-api/src/errors.js @@ -23,3 +23,16 @@ export class RecordNotFound extends Server.Failure { return RecordNotFoundErrorName } } + +export const DecodeBlockOperationErrorName = /** @type {const} */ ( + 'DecodeBlockOperationFailed' +) +export class DecodeBlockOperationFailed extends Server.Failure { + get reason() { + return this.message + } + + get name() { + return DecodeBlockOperationErrorName + } +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index ec9080616..51c78ede6 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -57,12 +57,6 @@ import { BlobAdd, BlobAddSuccess, BlobAddFailure, - BlobRemove, - BlobRemoveSuccess, - BlobRemoveFailure, - BlobList, - BlobListSuccess, - BlobListFailure, BlobAllocate, BlobAllocateSuccess, BlobAllocateFailure, @@ -192,8 +186,6 @@ export type { AllocationsStorage, BlobsStorage, TasksStorage, BlobAddInput } export interface Service extends StorefrontService, W3sService { blob: { add: ServiceMethod - remove: ServiceMethod - list: ServiceMethod } store: { add: ServiceMethod diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index ead5c752b..e6623e2d4 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -1,8 +1,10 @@ import { provide } from '@ucanto/server' import { Message } from '@ucanto/core' import { CAR } from '@ucanto/transport' -import * as Blob from '@web3-storage/capabilities/blob' +import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' +import * as HTTP from '@web3-storage/capabilities/http' import { conclude } from '@web3-storage/capabilities/ucan' +import { DecodeBlockOperationFailed } from '../errors.js' import * as API from '../types.js' /** @@ -15,8 +17,15 @@ export const ucanConcludeProvider = ({ tasksStorage, tasksScheduler, }) => - provide(conclude, async ({ capability }) => { - const messageCar = CAR.codec.decode(capability.nb.bytes) + provide(conclude, async ({ capability, invocation }) => { + const getBlockRes = await findBlock( + capability.nb.message, + invocation.iterateIPLDBlocks() + ) + if (getBlockRes.error) { + return getBlockRes + } + const messageCar = CAR.codec.decode(getBlockRes.ok) const message = Message.view({ root: messageCar.roots[0].cid, store: messageCar.blocks, @@ -30,6 +39,8 @@ export const ucanConcludeProvider = ({ throw new Error('receipt should exist') } + // TODO: Verify invocation exists failing with InvocationNotFoundForReceipt + // Store receipt const receiptPutRes = await receiptsStorage.put(receipt) if (receiptPutRes.error) { @@ -42,7 +53,6 @@ export const ucanConcludeProvider = ({ // Schedule `blob/accept` const ranInvocation = receipt.ran - // TODO: This actually needs the accept task!!!! // Get invocation const httpPutTaskGetRes = await tasksStorage.get(ranInvocation.link()) if (httpPutTaskGetRes.error) { @@ -52,27 +62,29 @@ export const ucanConcludeProvider = ({ // Schedule `blob/accept` if there is a `http/put` capability const scheduleRes = await Promise.all( httpPutTaskGetRes.ok.capabilities - .filter((cap) => cap.can === Blob.put.can) + .filter((cap) => cap.can === HTTP.put.can) .map(async (cap) => { - const blobAccept = await Blob.accept.invoke({ - issuer: id, - audience: id, - with: id.toDIDKey(), - nb: { - // @ts-expect-error blob exists in put - blob: cap.nb.blob, - exp: Number.MAX_SAFE_INTEGER, - }, - expiration: Infinity, - }).delegate() - + const blobAccept = await W3sBlob.accept + .invoke({ + issuer: id, + audience: id, + with: id.toDIDKey(), + nb: { + // @ts-expect-error blob exists in `http/put` but unknown type here + blob: cap.nb.blob, + exp: Number.MAX_SAFE_INTEGER, + }, + expiration: Infinity, + }) + .delegate() + return tasksScheduler.schedule(blobAccept) }) ) - const scheduleErrors = scheduleRes.filter(res => res.error) + const scheduleErrors = scheduleRes.filter((res) => res.error) if (scheduleErrors.length && scheduleErrors[0].error) { return { - error: scheduleErrors[0].error + error: scheduleErrors[0].error, } } @@ -80,3 +92,25 @@ export const ucanConcludeProvider = ({ ok: { time: Date.now() }, } }) + +/** + * @param {import('multiformats').UnknownLink} cid + * @param {IterableIterator>} blocks + * @returns {Promise>} + */ +export const findBlock = async (cid, blocks) => { + let bytes + for (const b of blocks) { + if (b.cid.equals(cid)) { + bytes = b.bytes + } + } + if (!bytes) { + return { + error: new DecodeBlockOperationFailed(`missing block: ${cid}`), + } + } + return { + ok: bytes, + } +} diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 0b8bd3c86..4df6ffa71 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -7,13 +7,16 @@ import { ed25519 } from '@ucanto/principal' import { CAR } from '@ucanto/transport' import { sha256 } from 'multiformats/hashes/sha2' import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' +import * as HTTPCapabilities from '@web3-storage/capabilities/http' import * as UCAN from '@web3-storage/capabilities/ucan' import { base64pad } from 'multiformats/bases/base64' import { provisionProvider } from '../helpers/utils.js' import { createServer, connect } from '../../src/lib.js' import { alice, bob, createSpace, registerSpace } from '../util.js' -import { BlobItemSizeExceededName } from '../../src/blob/lib.js' +import { BlobExceedsSizeLimitName } from '../../src/blob/lib.js' +import { findBlock } from '../../src/ucan/conclude.js' /** * @type {API.Tests} @@ -54,9 +57,10 @@ export const test = { } // Validate receipt - assert.ok(blobAdd.out.ok.claim) + assert.ok(blobAdd.out.ok.location) + assert.equal(blobAdd.out.ok.location['ucan/await'][0], '.out.ok.claim') assert.ok( - blobAdd.out.ok.claim['await/ok'].equals(blobAdd.fx.join?.link()) + blobAdd.out.ok.location['ucan/await'][1].equals(blobAdd.fx.join?.link()) ) assert.ok(blobAdd.fx.join) @@ -67,13 +71,13 @@ export const test = { const forkInvocations = blobAdd.fx.fork assert.equal(blobAdd.fx.fork.length, 3) const allocatefx = forkInvocations.find( - (fork) => fork.capabilities[0].can === BlobCapabilities.allocate.can + (fork) => fork.capabilities[0].can === W3sBlobCapabilities.allocate.can ) const allocateUcanConcludefx = forkInvocations.find( (fork) => fork.capabilities[0].can === UCAN.conclude.can ) const putfx = forkInvocations.find( - (fork) => fork.capabilities[0].can === BlobCapabilities.put.can + (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can ) if (!allocatefx || !allocateUcanConcludefx || !putfx) { throw new Error('effects not provided') @@ -81,19 +85,22 @@ export const test = { // validate facts exist for `http/put` assert.ok(putfx.facts.length) - const [{ bytes, did }] = putfx.facts - assert.ok(bytes) - assert.ok(did) + assert.ok(putfx.facts[0]['keys']) // Validate `http/put` invocation stored const httpPutGetTask = await context.tasksStorage.get(putfx.cid) assert.ok(httpPutGetTask.ok) - // validate scheduled allocate task ran an its receipt content - const messageCar = CAR.codec.decode( + // validate that scheduled allocate task executed and has its receipt content + const getBlockRes = await findBlock( // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.bytes + allocateUcanConcludefx.capabilities[0].nb.message, + allocateUcanConcludefx.iterateIPLDBlocks() ) + if (getBlockRes.error) { + throw new Error('receipt block should exist in invocation') + } + const messageCar = CAR.codec.decode(getBlockRes.ok) const message = Message.view({ root: messageCar.roots[0].cid, store: messageCar.blocks, @@ -108,7 +115,7 @@ export const test = { // @ts-expect-error receipt out is unknown assert.ok(receipt?.out.ok?.address) }, - 'blob/add executes allocation and returns effects for allocate (and its receipt) and accept, but not for put when blob stored': + 'blob/add executes allocation and returns effects for allocate (and its receipt) and accept, but not for put when blob stored': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -158,15 +165,20 @@ export const test = { (fork) => fork.capabilities[0].can === UCAN.conclude.can ) const putfx = forkInvocations.find( - (fork) => fork.capabilities[0].can === BlobCapabilities.put.can + (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can ) if (!allocateUcanConcludefx || !putfx) { throw new Error('effects not provided') } - const messageCar = CAR.codec.decode( + const getBlockRes = await findBlock( // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.bytes + allocateUcanConcludefx.capabilities[0].nb.message, + allocateUcanConcludefx.iterateIPLDBlocks() ) + if (getBlockRes.error) { + throw new Error('receipt block should exist in invocation') + } + const messageCar = CAR.codec.decode(getBlockRes.ok) const message = Message.view({ root: messageCar.roots[0].cid, store: messageCar.blocks, @@ -220,9 +232,11 @@ export const test = { // @ts-expect-error read only effect const thirdForkInvocations = thirdBlobAdd.fx.fork // no put effect anymore - assert.ok(!thirdForkInvocations.find( - (fork) => fork.capabilities[0].can === BlobCapabilities.put.can - )) + assert.ok( + !thirdForkInvocations.find( + (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can + ) + ) }, 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { @@ -257,7 +271,7 @@ export const test = { throw new Error('invocation should have failed') } assert.ok(blobAdd.out.error, 'invocation should have failed') - assert.equal(blobAdd.out.error.name, BlobItemSizeExceededName) + assert.equal(blobAdd.out.error.name, BlobExceedsSizeLimitName) }, 'blob/allocate allocates to space and returns presigned url': async ( assert, @@ -293,7 +307,7 @@ export const test = { }) // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ issuer: alice, audience: context.id, with: spaceDid, @@ -391,7 +405,7 @@ export const test = { }) // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ issuer: alice, audience: context.id, with: spaceDid, @@ -469,7 +483,7 @@ export const test = { }) // invoke `service/blob/allocate` capabilities on alice space - const aliceServiceBlobAllocate = BlobCapabilities.allocate.invoke({ + const aliceServiceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ issuer: alice, audience: context.id, with: aliceSpaceDid, @@ -510,7 +524,7 @@ export const test = { assert.equal(goodPut.status, 200, await goodPut.text()) // invoke `service/blob/allocate` capabilities on bob space - const bobServiceBlobAllocate = BlobCapabilities.allocate.invoke({ + const bobServiceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ issuer: bob, audience: context.id, with: bobSpaceDid, @@ -577,7 +591,7 @@ export const test = { }) // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ issuer: alice, audience: context.id, with: spaceDid, @@ -654,7 +668,7 @@ export const test = { }) // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ issuer: alice, audience: context.id, with: spaceDid, @@ -729,7 +743,7 @@ export const test = { }) // invoke `service/blob/allocate` - const serviceBlobAllocate = BlobCapabilities.allocate.invoke({ + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ issuer: alice, audience: context.id, with: spaceDid, @@ -784,7 +798,7 @@ export const test = { return Promise.resolve({ ok: {}, }) - } + }, }, }), }) @@ -817,15 +831,20 @@ export const test = { (fork) => fork.capabilities[0].can === UCAN.conclude.can ) const putfx = forkInvocations.find( - (fork) => fork.capabilities[0].can === BlobCapabilities.put.can + (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can ) if (!allocateUcanConcludefx || !putfx) { throw new Error('effects not provided') } - const blobAllocateMessageCar = CAR.codec.decode( + const getBlockRes = await findBlock( // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.bytes + allocateUcanConcludefx.capabilities[0].nb.message, + allocateUcanConcludefx.iterateIPLDBlocks() ) + if (getBlockRes.error) { + throw new Error('receipt block should exist in invocation') + } + const blobAllocateMessageCar = CAR.codec.decode(getBlockRes.ok) const blobAllocateMessage = Message.view({ root: blobAllocateMessageCar.roots[0].cid, store: blobAllocateMessageCar.blocks, @@ -851,11 +870,10 @@ export const test = { assert.equal(goodPut.status, 200, await goodPut.text()) // Create `http/put` receipt - /** @type {{ bytes: Uint8Array, did: string }} */ - // @ts-expect-error facts are unknown - const [{ bytes, did }] = putfx.facts - const putSubject = ed25519.decode(bytes) - const httpPut = BlobCapabilities.put.invoke({ + const keys = putfx.facts[0]['keys'] + // @ts-expect-error Argument of type 'unknown' is not assignable to parameter of type 'SignerArchive<`did:${string}:${string}`, SigAlg>' + const putSubject = ed25519.from(keys) + const httpPut = HTTPCapabilities.put.invoke({ issuer: putSubject, audience: putSubject, with: putSubject.toDIDKey(), @@ -864,7 +882,7 @@ export const test = { content, size, }, - address + address, }, facts: putfx.facts, expiration: Infinity, @@ -880,6 +898,8 @@ export const test = { }) const message = await Message.build({ receipts: [httpPutReceipt] }) const messageCar = await CAR.outbound.encode(message) + const bytes = new Uint8Array(messageCar.body) + const messageLink = await CAR.codec.link(bytes) // Invoke `ucan/conclude` with `http/put` receipt const httpPutConcludeInvocation = UCAN.conclude.invoke({ @@ -887,10 +907,14 @@ export const test = { audience: context.id, with: alice.did(), nb: { - bytes: messageCar.body, + message: messageLink, }, expiration: Infinity, }) + httpPutConcludeInvocation.attach({ + bytes, + cid: messageLink, + }) const ucanConclude = await httpPutConcludeInvocation.execute(connection) if (!ucanConclude.out.ok) { throw new Error('invocation failed', { cause: blobAdd }) @@ -899,12 +923,13 @@ export const test = { // verify accept was scheduled const blobAcceptInvocation = await taskScheduled.promise assert.ok(blobAcceptInvocation) + assert.equal(blobAdd.out.ok.location['ucan/await'][0], '.out.ok.claim') assert.ok( - blobAdd.out.ok.claim['await/ok'].equals(blobAcceptInvocation.cid) + blobAdd.out.ok.location['ucan/await'][1].equals( + blobAcceptInvocation.cid + ) ) assert.ok(blobAdd.fx.join?.link().equals(blobAcceptInvocation.cid)) }, // TODO: Blob accept - // TODO: list - // TODO: remove } From ab02d083673f127fdd83169b488d7905334b147b Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 5 Apr 2024 18:16:36 +0200 Subject: [PATCH 06/27] fix: minor tweaks --- packages/capabilities/src/types.ts | 18 ++--------------- packages/upload-api/src/types/blob.ts | 7 ------- .../test/storage/allocations-storage.js | 20 ------------------- 3 files changed, 2 insertions(+), 43 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 2177b9958..0d28e5e72 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -468,29 +468,14 @@ export interface BlobAddSuccess { export interface BlobExceedsSizeLimit extends Ucanto.Failure { name: 'BlobExceedsSizeLimit' } +// TODO: We should type the store errors and add them here, instead of Ucanto.Failure export type BlobAddFailure = BlobExceedsSizeLimit | Ucanto.Failure -// Blob remove -export interface BlobRemoveSuccess { - size: number -} - -export interface BlobItemNotFound extends Ucanto.Failure { - name: 'BlobItemNotFound' -} - -// TODO: Add more errors from stores -export type BlobRemoveFailure = BlobItemNotFound | Ucanto.Failure - -// Blob list -export interface BlobListSuccess extends ListResponse {} export interface BlobListItem { blob: BlobModel insertedAt: ISO8601Date } -export type BlobListFailure = Ucanto.Failure - // Blob allocate export interface BlobAllocateSuccess { size: number @@ -520,6 +505,7 @@ export interface BlobAcceptSuccess { claim: Link } +// TODO: We should type the store errors and add them here, instead of Ucanto.Failure export type BlobAcceptFailure = BlobItemNotFound | Ucanto.Failure // Store diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index 8adcd03f8..3bcf18bfa 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -8,12 +8,10 @@ import type { import { BlobMultihash, BlobListItem, - BlobRemoveSuccess, } from '@web3-storage/capabilities/types' import { RecordKeyConflict, - RecordNotFound, ListOptions, ListResponse, } from '../types.js' @@ -34,11 +32,6 @@ export interface AllocationsStorage { insert: ( item: BlobAddInput ) => Promise> - /** Removes an item from the table but fails if the item does not exist. */ - remove: ( - space: DID, - blobMultihash: BlobMultihash - ) => Promise> list: ( space: DID, options?: ListOptions diff --git a/packages/upload-api/test/storage/allocations-storage.js b/packages/upload-api/test/storage/allocations-storage.js index 67946a97b..12810675a 100644 --- a/packages/upload-api/test/storage/allocations-storage.js +++ b/packages/upload-api/test/storage/allocations-storage.js @@ -60,26 +60,6 @@ export class AllocationsStorage { return { ok: !!item } } - /** - * @param {Types.DID} space - * @param {Uint8Array} blobMultihash - * @returns {ReturnType} - */ - async remove(space, blobMultihash) { - const item = this.items.find( - (i) => i.space === space && equals(i.blob.content, blobMultihash) - ) - if (!item) { - return { error: { name: 'RecordNotFound', message: 'record not found' } } - } - this.items = this.items.filter((i) => i !== item) - return { - ok: { - size: item.blob.size, - }, - } - } - /** * @param {Types.DID} space * @param {Types.ListOptions} options From 89311243ac16aae05e0a6ae373909fc4e1c9b248 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 5 Apr 2024 18:51:28 +0200 Subject: [PATCH 07/27] chore: add location claim --- .../capabilities/src/web3.storage/blob.js | 1 + packages/upload-api/package.json | 1 + packages/upload-api/src/blob/accept.js | 58 +++++++++++++++---- pnpm-lock.yaml | 3 + 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/packages/capabilities/src/web3.storage/blob.js b/packages/capabilities/src/web3.storage/blob.js index 56188c847..206575955 100644 --- a/packages/capabilities/src/web3.storage/blob.js +++ b/packages/capabilities/src/web3.storage/blob.js @@ -78,6 +78,7 @@ export const accept = capability({ * Blob to accept. */ blob: blobStruct, + // TODO: @gozala suggested we could use content length from `httt/put` to figure out size and not need to pass the blob here /** * Expiration.. */ diff --git a/packages/upload-api/package.json b/packages/upload-api/package.json index 25cdfbb9f..7a3156a3e 100644 --- a/packages/upload-api/package.json +++ b/packages/upload-api/package.json @@ -179,6 +179,7 @@ "@ucanto/validator": "^9.0.2", "@web3-storage/access": "workspace:^", "@web3-storage/capabilities": "workspace:^", + "@web3-storage/content-claims": "^4.0.4", "@web3-storage/did-mailto": "workspace:^", "@web3-storage/filecoin-api": "workspace:^", "multiformats": "^12.1.2", diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index f7a26af44..d06b9eda2 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -1,5 +1,10 @@ import * as Server from '@ucanto/server' import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' +import { Assert } from '@web3-storage/content-claims/capability' +import { create as createLink } from 'multiformats/link' +import { Digest } from 'multiformats/hashes/digest' +import { sha256 } from 'multiformats/hashes/sha2' +import { CAR } from '@ucanto/core' import * as API from '../types.js' import { BlobItemNotFound } from './lib.js' @@ -8,21 +13,50 @@ import { BlobItemNotFound } from './lib.js' * @returns {API.ServiceMethod} */ export function blobAcceptProvider(context) { - return Server.provide(W3sBlob.accept, async ({ capability }) => { - const { blob } = capability.nb - // If blob is not stored, we must fail - const hasBlob = await context.blobsStorage.has(blob.content) - if (hasBlob.error) { - return { - error: new BlobItemNotFound(), + return Server.provideAdvanced({ + capability: W3sBlob.accept, + handler: async ({ capability }) => { + const { blob } = capability.nb + // If blob is not stored, we must fail + const hasBlob = await context.blobsStorage.has(blob.content) + if (hasBlob.error) { + return { + error: new BlobItemNotFound(), + } } - } - // TODO: Set bucket name - // TODO: return content commitment + const digest = new Digest(sha256.code, 32, blob.content, blob.content) + const content = createLink(CAR.code, digest) + const w3link = `https://w3s.link/ipfs/${content.toString()}` + + // TODO: Set bucket name + // TODO: return content commitment + const locationClaim = await Assert.location + .invoke({ + issuer: context.id, + // TODO: we need user agent DID + audience: context.id, + with: context.id.toDIDKey(), + nb: { + content, + location: [ + // @ts-expect-error Type 'string' is not assignable to type '`${string}:${string}`' + w3link + ] + }, + expiration: Infinity, + }) + .delegate() + // TODO: we need to support multihash in claims, or specify hardcoded codec + + // Create result object + /** @type {API.OkBuilder} */ + const result = Server.ok({ + claim: locationClaim.cid + }) - return { - error: new BlobItemNotFound(), + return result + .fork(locationClaim) } }) } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 566bb02aa..d4f4b5479 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -403,6 +403,9 @@ importers: '@web3-storage/capabilities': specifier: workspace:^ version: link:../capabilities + '@web3-storage/content-claims': + specifier: ^4.0.4 + version: 4.0.4 '@web3-storage/did-mailto': specifier: workspace:^ version: link:../did-mailto From 9fac96cb612673e505af86983d1f1355ae167eea Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 8 Apr 2024 09:55:03 +0200 Subject: [PATCH 08/27] chore: address review --- packages/capabilities/src/blob.js | 4 +-- packages/capabilities/src/http.js | 31 +++++++--------- packages/capabilities/src/utils.js | 36 ++++++++++++++++++- .../capabilities/src/web3.storage/blob.js | 6 ++-- packages/upload-api/src/blob/add.js | 18 ++++++---- packages/upload-api/src/ucan/conclude.js | 5 +-- packages/upload-api/test/handlers/blob.js | 15 ++++---- 7 files changed, 75 insertions(+), 40 deletions(-) diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index a9108dcd4..e30b604fc 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -36,7 +36,7 @@ export const blob = capability({ /** * Blob description for being ingested by the service. */ -export const blobStruct = Schema.struct({ +export const blobModel = Schema.struct({ /** * A multihash digest of the blob payload bytes, uniquely identifying blob. */ @@ -66,7 +66,7 @@ export const add = capability({ /** * Blob to allocate on the space. */ - blob: blobStruct, + blob: blobModel, }), derives: equalBlob, }) diff --git a/packages/capabilities/src/http.js b/packages/capabilities/src/http.js index cf171ee4c..b6220307d 100644 --- a/packages/capabilities/src/http.js +++ b/packages/capabilities/src/http.js @@ -9,8 +9,8 @@ * @module */ import { capability, Schema, ok } from '@ucanto/validator' -import { blobStruct } from './blob.js' -import { equal, equalBlob, equalWith, SpaceDID, and } from './utils.js' +import { blobModel } from './blob.js' +import { equal, equalBody, equalWith, SpaceDID, and } from './utils.js' /** * `http/put` capability invocation MAY be performed by any agent on behalf of the subject. @@ -26,29 +26,24 @@ export const put = capability({ with: SpaceDID, nb: Schema.struct({ /** - * Blob to allocate on the space. + * BodyBlob to allocate on the space. */ - blob: blobStruct, + body: blobModel, /** - * Blob to accept. + * HTTP(S) location that can receive blob content via HTTP PUT request. */ - address: Schema.struct({ - /** - * HTTP(S) location that can receive blob content via HTTP PUT request. - */ - url: Schema.string(), - /** - * HTTP headers. - */ - headers: Schema.unknown(), - }).optional(), + url: Schema.string(), + /** + * HTTP headers. + */ + headers: Schema.dictionary({ value: Schema.string() }), }), derives: (claim, from) => { return ( and(equalWith(claim, from)) || - and(equalBlob(claim, from)) || - and(equal(claim.nb.address?.url, from.nb.address, 'url')) || - and(equal(claim.nb.address?.headers, from.nb.address, 'headers')) || + and(equalBody(claim, from)) || + and(equal(claim.nb.url, from.nb, 'url')) || + and(equal(claim.nb.headers, from.nb, 'headers')) || ok({}) ) }, diff --git a/packages/capabilities/src/utils.js b/packages/capabilities/src/utils.js index 9e61fada9..f82ae38d3 100644 --- a/packages/capabilities/src/utils.js +++ b/packages/capabilities/src/utils.js @@ -88,7 +88,7 @@ export const equalLink = (claimed, delegated) => { } /** - * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"web3.storage/blob/allocate"|"web3.storage/blob/accept"|"http/put", Types.URI<'did:'>, {blob: { content: Uint8Array, size: number }}>} T + * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"web3.storage/blob/allocate"|"web3.storage/blob/accept", Types.URI<'did:'>, {blob: { content: Uint8Array, size: number }}>} T * @param {T} claimed * @param {T} delegated * @returns {Types.Result<{}, Types.Failure>} @@ -121,6 +121,40 @@ export const equalBlob = (claimed, delegated) => { } } +/** + * @template {Types.ParsedCapability<"http/put", Types.URI<'did:'>, {body: { content: Uint8Array, size: number }}>} T + * @param {T} claimed + * @param {T} delegated + * @returns {Types.Result<{}, Types.Failure>} + */ +export const equalBody = (claimed, delegated) => { + if (claimed.with !== delegated.with) { + return fail( + `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` + ) + } else if ( + delegated.nb.body.content && + !equals(delegated.nb.body.content, claimed.nb.body.content) + ) { + return fail( + `Link ${ + claimed.nb.body.content ? `${claimed.nb.body.content}` : '' + } violates imposed ${delegated.nb.body.content} constraint.` + ) + } else if ( + claimed.nb.body.size !== undefined && + delegated.nb.body.size !== undefined + ) { + return claimed.nb.body.size > delegated.nb.body.size + ? fail( + `Size constraint violation: ${claimed.nb.body.size} > ${delegated.nb.body.size}` + ) + : ok({}) + } else { + return ok({}) + } +} + /** * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"blob/allocate"|"blob/accept"|"http/put", Types.URI<'did:'>, {content: Uint8Array}>} T * @param {T} claimed diff --git a/packages/capabilities/src/web3.storage/blob.js b/packages/capabilities/src/web3.storage/blob.js index 206575955..9227c6365 100644 --- a/packages/capabilities/src/web3.storage/blob.js +++ b/packages/capabilities/src/web3.storage/blob.js @@ -1,5 +1,5 @@ import { capability, Schema, Link, ok, fail } from '@ucanto/validator' -import { blobStruct } from '../blob.js' +import { blobModel } from '../blob.js' import { equalBlob, equalWith, @@ -41,7 +41,7 @@ export const allocate = capability({ /** * Blob to allocate on the space. */ - blob: blobStruct, + blob: blobModel, /** * The Link for an Add Blob task, that caused an allocation */ @@ -77,7 +77,7 @@ export const accept = capability({ /** * Blob to accept. */ - blob: blobStruct, + blob: blobModel, // TODO: @gozala suggested we could use content length from `httt/put` to figure out size and not need to pass the blob here /** * Expiration.. diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 300d097ea..667fa94fa 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -41,10 +41,10 @@ export function blobAddProvider(context) { // of the `http/put` invocation. That way anyone with blob content // could perform the invocation and issue receipt by deriving same // principal - const putSubject = await ed25519.derive(blob.content.slice(0, 32)) + const blobProvider = await ed25519.derive(blob.content.slice(0, 32)) const facts = [ { - keys: putSubject.toArchive(), + keys: blobProvider.toArchive(), }, ] @@ -78,6 +78,7 @@ export function blobAddProvider(context) { // Get receipt for `blob/allocate` if available, or schedule invocation if not const allocatedGetRes = await allocationsStorage.get(space, blob.content) let blobAllocateReceipt + /** @type {API.BlobAddress | undefined} */ let blobAllocateOutAddress // If already allocated, just get the allocate receipt // and the addresses if still pending to receive blob @@ -92,6 +93,8 @@ export function blobAddProvider(context) { const blobHasRes = await context.blobsStorage.has(blob.content) if (blobHasRes.error) { return blobHasRes + // If still not stored, keep the allocate address to signal to the client + // that bytes MUST be sent through the `http/put` effect } else if (!blobHasRes.ok) { // @ts-expect-error receipt type is unknown blobAllocateOutAddress = blobAllocateReceipt.out.ok.address @@ -145,12 +148,13 @@ export function blobAddProvider(context) { // the blob is still not stored if (blobAllocateOutAddress) { const blobPut = HTTP.put.invoke({ - issuer: putSubject, - audience: putSubject, - with: putSubject.toDIDKey(), + issuer: blobProvider, + audience: blobProvider, + with: blobProvider.toDIDKey(), nb: { - blob, - address: blobAllocateOutAddress, + body: blob, + url: blobAllocateOutAddress.url, + headers: blobAllocateOutAddress.headers }, facts, expiration: Infinity, diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index e6623e2d4..136a7cef7 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -70,8 +70,8 @@ export const ucanConcludeProvider = ({ audience: id, with: id.toDIDKey(), nb: { - // @ts-expect-error blob exists in `http/put` but unknown type here - blob: cap.nb.blob, + // @ts-expect-error body exists in `http/put` but unknown type here + blob: cap.nb.body, exp: Number.MAX_SAFE_INTEGER, }, expiration: Infinity, @@ -81,6 +81,7 @@ export const ucanConcludeProvider = ({ return tasksScheduler.schedule(blobAccept) }) ) + const scheduleErrors = scheduleRes.filter((res) => res.error) if (scheduleErrors.length && scheduleErrors[0].error) { return { diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 4df6ffa71..bb465fea4 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -872,17 +872,18 @@ export const test = { // Create `http/put` receipt const keys = putfx.facts[0]['keys'] // @ts-expect-error Argument of type 'unknown' is not assignable to parameter of type 'SignerArchive<`did:${string}:${string}`, SigAlg>' - const putSubject = ed25519.from(keys) + const blobProvider = ed25519.from(keys) const httpPut = HTTPCapabilities.put.invoke({ - issuer: putSubject, - audience: putSubject, - with: putSubject.toDIDKey(), + issuer: blobProvider, + audience: blobProvider, + with: blobProvider.toDIDKey(), nb: { - blob: { + body: { content, size, }, - address, + url: address.url, + headers: address.headers }, facts: putfx.facts, expiration: Infinity, @@ -890,7 +891,7 @@ export const test = { const httpPutDelegation = await httpPut.delegate() const httpPutReceipt = await Receipt.issue({ - issuer: putSubject, + issuer: blobProvider, ran: httpPutDelegation.cid, result: { ok: {}, From 730b6016864cb1242abb1dbc85f63e0df9aeca1b Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 8 Apr 2024 15:45:17 +0200 Subject: [PATCH 09/27] chore: address review --- packages/capabilities/src/types.ts | 1 + packages/upload-api/src/blob/add.js | 1 + packages/upload-api/src/blob/allocate.js | 26 ++++++++++++------- packages/upload-api/src/types/blob.ts | 6 ++++- .../upload-api/test/storage/blobs-storage.js | 9 +++---- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 0d28e5e72..4a9deacb6 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -479,6 +479,7 @@ export interface BlobListItem { // Blob allocate export interface BlobAllocateSuccess { size: number + expiresAt?: ISO8601Date address?: BlobAddress } diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 667fa94fa..483370b62 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -83,6 +83,7 @@ export function blobAddProvider(context) { // If already allocated, just get the allocate receipt // and the addresses if still pending to receive blob if (allocatedGetRes.ok) { + // TODO: Check expires const receiptGet = await context.receiptsStorage.get(allocatefx.link()) if (receiptGet.error) { return receiptGet diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index bf78d5691..ea349cf22 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -53,7 +53,9 @@ export function blobAllocateProvider(context) { if (allocationInsert.error) { // if the insert failed with conflict then this item has already been // added to the space and there is no allocation change. + // If record exists but is expired, it can be re-written if (allocationInsert.error.name === 'RecordKeyConflict') { + // TODO: Updates to new URL and expiration if expired? return { ok: { size: 0 }, } @@ -63,6 +65,19 @@ export function blobAllocateProvider(context) { } } + // Get presigned URL for the write target + const expiresIn = 60 * 60 * 24 // 1 day + const createUploadUrl = await context.blobsStorage.createUploadUrl( + blob.content, + blob.size, + expiresIn + ) + if (createUploadUrl.error) { + return { + error: new Server.Failure('failed to provide presigned url'), + } + } + // Check if blob already exists const hasBlobStore = await context.blobsStorage.has(blob.content) if (hasBlobStore.error) { @@ -76,16 +91,6 @@ export function blobAllocateProvider(context) { } } - // Get presigned URL for the write target - const createUploadUrl = await context.blobsStorage.createUploadUrl( - blob.content, - blob.size - ) - if (createUploadUrl.error) { - return { - error: new Server.Failure('failed to provide presigned url'), - } - } const address = { url: createUploadUrl.ok.url.toString(), headers: createUploadUrl.ok.headers, @@ -95,6 +100,7 @@ export function blobAllocateProvider(context) { ok: { size: blob.size, address, + expiresAt: (new Date(Date.now() + expiresIn)).toISOString() }, } } diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index 3bcf18bfa..405c4fe66 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -61,7 +61,11 @@ export interface BlobsStorage { has: (content: BlobMultihash) => Promise> createUploadUrl: ( content: BlobMultihash, - size: number + size: number, + /** + * The number of seconds before the presigned URL expires + */ + expiresIn: number ) => Promise< Result< { diff --git a/packages/upload-api/test/storage/blobs-storage.js b/packages/upload-api/test/storage/blobs-storage.js index 51135e809..cf821c918 100644 --- a/packages/upload-api/test/storage/blobs-storage.js +++ b/packages/upload-api/test/storage/blobs-storage.js @@ -99,7 +99,6 @@ export class BlobsStorage { secretAccessKey = 'secret', bucket = 'my-bucket', region = 'eu-central-1', - expires, }) { this.server = server this.baseURL = url @@ -107,7 +106,6 @@ export class BlobsStorage { this.secretAccessKey = secretAccessKey this.bucket = bucket this.region = region - this.expires = expires this.content = content } @@ -126,9 +124,10 @@ export class BlobsStorage { /** * @param {Uint8Array} multihash * @param {number} size + * @param {number} expiresIn */ - async createUploadUrl(multihash, size) { - const { bucket, expires, accessKeyId, secretAccessKey, region, baseURL } = + async createUploadUrl(multihash, size, expiresIn) { + const { bucket, accessKeyId, secretAccessKey, region, baseURL } = this const encodedMultihash = base58btc.encode(multihash) const multihashDigest = digestDecode(multihash) @@ -144,7 +143,7 @@ export class BlobsStorage { key: `${encodedMultihash}/${encodedMultihash}.blob`, checksum, bucket, - expires, + expires: expiresIn, }) const url = new URL(baseURL) From 92500c68f066c9764e7ea64e0749ab630f1c5213 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 8 Apr 2024 16:08:51 +0200 Subject: [PATCH 10/27] chore: add await error --- packages/capabilities/src/types.ts | 7 ++++++- packages/upload-api/src/blob/add.js | 8 ++++++-- packages/upload-api/src/blob/lib.js | 28 ++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 4a9deacb6..f5d880dbe 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -468,8 +468,13 @@ export interface BlobAddSuccess { export interface BlobExceedsSizeLimit extends Ucanto.Failure { name: 'BlobExceedsSizeLimit' } + +export interface AwaitError extends Ucanto.Failure { + name: 'AwaitError' +} + // TODO: We should type the store errors and add them here, instead of Ucanto.Failure -export type BlobAddFailure = BlobExceedsSizeLimit | Ucanto.Failure +export type BlobAddFailure = BlobExceedsSizeLimit | AwaitError | Ucanto.Failure export interface BlobListItem { blob: BlobModel diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 483370b62..9a2477692 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -8,7 +8,7 @@ import * as HTTP from '@web3-storage/capabilities/http' import * as UCAN from '@web3-storage/capabilities/ucan' import * as API from '../types.js' -import { BlobExceedsSizeLimit } from './lib.js' +import { BlobExceedsSizeLimit, AwaitError } from './lib.js' /** * @param {API.BlobServiceContext} context @@ -107,7 +107,11 @@ export function blobAddProvider(context) { const allocateRes = await blobAllocate.execute(getServiceConnection()) if (allocateRes.out.error) { return { - error: allocateRes.out.error, + error: new AwaitError({ + cause: allocateRes.out.error, + at: 'ucan/wait', + reference: ['.out.ok', allocatefx.cid] + }) } } // If this is a new allocation, `http/put` effect should be returned with address diff --git a/packages/upload-api/src/blob/lib.js b/packages/upload-api/src/blob/lib.js index 0e4b1b86a..663d93e68 100644 --- a/packages/upload-api/src/blob/lib.js +++ b/packages/upload-api/src/blob/lib.js @@ -54,3 +54,31 @@ export class BlobExceedsSizeLimit extends Failure { } } } + +export const AwaitErrorName = 'AwaitError' +export class AwaitError extends Failure { + /** + * @param {object} source + * @param {string} source.at - argument path that referenced failed `await` + * @param {[selector: string, task: import('@ucanto/interface').UnknownLink]} source.reference - awaited reference that failed + * @param {import('@ucanto/interface').Failure} source.cause - error that caused referenced `await` to fail + */ + constructor({ at, reference, cause }) { + super() + this.at = at + this.reference = reference + this.cause = cause + } + describe() { + const [selector, task] = this.reference + return `Awaited (${selector} ${task}) reference at ${this.at} has failed:\n${this.cause}` + } + get name() { + return AwaitErrorName + } + toJSON() { + return { + ...super.toJSON(), + } + } +} From c88f63440ba8e0d890e9a8efffa69baa81d317b8 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 9 Apr 2024 13:47:08 +0200 Subject: [PATCH 11/27] chore: update based on https://github.com/web3-storage/specs/pull/117 --- packages/capabilities/src/blob.js | 8 +- packages/capabilities/src/http.js | 17 +++- packages/capabilities/src/types.ts | 9 +-- packages/capabilities/src/utils.js | 20 ++--- .../capabilities/src/web3.storage/blob.js | 15 ++-- packages/upload-api/src/blob/accept.js | 17 ++-- packages/upload-api/src/blob/add.js | 28 +++---- packages/upload-api/src/blob/allocate.js | 7 +- packages/upload-api/src/blob/lib.js | 12 +-- packages/upload-api/src/types/blob.ts | 15 +--- packages/upload-api/src/ucan/conclude.js | 1 + packages/upload-api/test/handlers/blob.js | 77 +++++++++---------- .../test/storage/allocations-storage.js | 6 +- .../upload-api/test/storage/blobs-storage.js | 3 +- 14 files changed, 121 insertions(+), 114 deletions(-) diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index e30b604fc..4ef350b0d 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -36,11 +36,11 @@ export const blob = capability({ /** * Blob description for being ingested by the service. */ -export const blobModel = Schema.struct({ +export const content = Schema.struct({ /** * A multihash digest of the blob payload bytes, uniquely identifying blob. */ - content: Schema.bytes(), + digest: Schema.bytes(), /** * Size of the Blob file to be stored. Service will provision write target * for this exact size. Attempt to write a larger Blob file will fail. @@ -51,7 +51,7 @@ export const blobModel = Schema.struct({ /** * `blob/add` capability allows agent to store a Blob into a (memory) space * identified by did:key in the `with` field. Agent must precompute Blob locally - * and provide it's multihash and size using `nb.content` and `nb.size` fields, allowing + * and provide it's multihash and size using `nb.blob` field, allowing * a service to provision a write location for the agent to PUT or POST desired * Blob into. */ @@ -66,7 +66,7 @@ export const add = capability({ /** * Blob to allocate on the space. */ - blob: blobModel, + blob: content, }), derives: equalBlob, }) diff --git a/packages/capabilities/src/http.js b/packages/capabilities/src/http.js index b6220307d..5f8d5d8c3 100644 --- a/packages/capabilities/src/http.js +++ b/packages/capabilities/src/http.js @@ -9,7 +9,7 @@ * @module */ import { capability, Schema, ok } from '@ucanto/validator' -import { blobModel } from './blob.js' +import { content } from './blob.js' import { equal, equalBody, equalWith, SpaceDID, and } from './utils.js' /** @@ -28,7 +28,20 @@ export const put = capability({ /** * BodyBlob to allocate on the space. */ - body: blobModel, + body: content, + // TODO: what should be used? + /** + * HTTP(S) location that can receive blob content via HTTP PUT request. + */ + // url: Schema.struct({ + // 'ucan/await': Schema.unknown(), + // }).optional(), + // /** + // * HTTP headers. + // */ + // headers: Schema.struct({ + // 'ucan/await': Schema.unknown(), + // }).optional(), /** * HTTP(S) location that can receive blob content via HTTP PUT request. */ diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index f5d880dbe..cd18c02bf 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -454,14 +454,14 @@ export type BlobAccept = InferInvokedCapability export type BlobMultihash = Uint8Array export interface BlobModel { - content: BlobMultihash + digest: BlobMultihash size: number } // Blob add export interface BlobAddSuccess { - location: { - 'ucan/await': ['.out.ok.claim', Link] + site: { + 'ucan/await': ['.out.ok.site', Link] } } @@ -484,7 +484,6 @@ export interface BlobListItem { // Blob allocate export interface BlobAllocateSuccess { size: number - expiresAt?: ISO8601Date address?: BlobAddress } @@ -508,7 +507,7 @@ export type BlobAllocateFailure = // Blob accept export interface BlobAcceptSuccess { - claim: Link + site: Link } // TODO: We should type the store errors and add them here, instead of Ucanto.Failure diff --git a/packages/capabilities/src/utils.js b/packages/capabilities/src/utils.js index f82ae38d3..d15afd523 100644 --- a/packages/capabilities/src/utils.js +++ b/packages/capabilities/src/utils.js @@ -88,7 +88,7 @@ export const equalLink = (claimed, delegated) => { } /** - * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"web3.storage/blob/allocate"|"web3.storage/blob/accept", Types.URI<'did:'>, {blob: { content: Uint8Array, size: number }}>} T + * @template {Types.ParsedCapability<"blob/add"|"blob/remove"|"web3.storage/blob/allocate"|"web3.storage/blob/accept", Types.URI<'did:'>, {blob: { digest: Uint8Array, size: number }}>} T * @param {T} claimed * @param {T} delegated * @returns {Types.Result<{}, Types.Failure>} @@ -99,13 +99,13 @@ export const equalBlob = (claimed, delegated) => { `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` ) } else if ( - delegated.nb.blob.content && - !equals(delegated.nb.blob.content, claimed.nb.blob.content) + delegated.nb.blob.digest && + !equals(delegated.nb.blob.digest, claimed.nb.blob.digest) ) { return fail( `Link ${ - claimed.nb.blob.content ? `${claimed.nb.blob.content}` : '' - } violates imposed ${delegated.nb.blob.content} constraint.` + claimed.nb.blob.digest ? `${claimed.nb.blob.digest}` : '' + } violates imposed ${delegated.nb.blob.digest} constraint.` ) } else if ( claimed.nb.blob.size !== undefined && @@ -122,7 +122,7 @@ export const equalBlob = (claimed, delegated) => { } /** - * @template {Types.ParsedCapability<"http/put", Types.URI<'did:'>, {body: { content: Uint8Array, size: number }}>} T + * @template {Types.ParsedCapability<"http/put", Types.URI<'did:'>, {body: { digest: Uint8Array, size: number }}>} T * @param {T} claimed * @param {T} delegated * @returns {Types.Result<{}, Types.Failure>} @@ -133,13 +133,13 @@ export const equalBody = (claimed, delegated) => { `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` ) } else if ( - delegated.nb.body.content && - !equals(delegated.nb.body.content, claimed.nb.body.content) + delegated.nb.body.digest && + !equals(delegated.nb.body.digest, claimed.nb.body.digest) ) { return fail( `Link ${ - claimed.nb.body.content ? `${claimed.nb.body.content}` : '' - } violates imposed ${delegated.nb.body.content} constraint.` + claimed.nb.body.digest ? `${claimed.nb.body.digest}` : '' + } violates imposed ${delegated.nb.body.digest} constraint.` ) } else if ( claimed.nb.body.size !== undefined && diff --git a/packages/capabilities/src/web3.storage/blob.js b/packages/capabilities/src/web3.storage/blob.js index 9227c6365..4817f033e 100644 --- a/packages/capabilities/src/web3.storage/blob.js +++ b/packages/capabilities/src/web3.storage/blob.js @@ -1,5 +1,5 @@ import { capability, Schema, Link, ok, fail } from '@ucanto/validator' -import { blobModel } from '../blob.js' +import { content } from '../blob.js' import { equalBlob, equalWith, @@ -41,7 +41,7 @@ export const allocate = capability({ /** * Blob to allocate on the space. */ - blob: blobModel, + blob: content, /** * The Link for an Add Blob task, that caused an allocation */ @@ -77,12 +77,17 @@ export const accept = capability({ /** * Blob to accept. */ - blob: blobModel, - // TODO: @gozala suggested we could use content length from `httt/put` to figure out size and not need to pass the blob here + blob: content, /** - * Expiration.. + * Expiration of location site. */ exp: Schema.integer(), + /** + * DID of the user space where allocation took place + */ + // TODO: space + // space: SpaceDID, + // TODO: _put? }), derives: (claim, from) => { const result = equalBlob(claim, from) diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index d06b9eda2..c35924d25 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -18,14 +18,15 @@ export function blobAcceptProvider(context) { handler: async ({ capability }) => { const { blob } = capability.nb // If blob is not stored, we must fail - const hasBlob = await context.blobsStorage.has(blob.content) + const hasBlob = await context.blobsStorage.has(blob.digest) if (hasBlob.error) { return { error: new BlobItemNotFound(), } } - const digest = new Digest(sha256.code, 32, blob.content, blob.content) + // TODO: we need to support multihash in claims, or specify hardcoded codec + const digest = new Digest(sha256.code, 32, blob.digest, blob.digest) const content = createLink(CAR.code, digest) const w3link = `https://w3s.link/ipfs/${content.toString()}` @@ -41,22 +42,20 @@ export function blobAcceptProvider(context) { content, location: [ // @ts-expect-error Type 'string' is not assignable to type '`${string}:${string}`' - w3link - ] + w3link, + ], }, expiration: Infinity, }) .delegate() - // TODO: we need to support multihash in claims, or specify hardcoded codec // Create result object /** @type {API.OkBuilder} */ const result = Server.ok({ - claim: locationClaim.cid + site: locationClaim.cid, }) - return result - .fork(locationClaim) - } + return result.fork(locationClaim) + }, }) } diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 9a2477692..ff8c9613b 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -37,11 +37,11 @@ export function blobAddProvider(context) { } } - // We derive principal from the content multihash to be an audience - // of the `http/put` invocation. That way anyone with blob content + // We derive principal from the blob multihash to be an audience + // of the `http/put` invocation. That way anyone with blob digest // could perform the invocation and issue receipt by deriving same // principal - const blobProvider = await ed25519.derive(blob.content.slice(0, 32)) + const blobProvider = await ed25519.derive(blob.digest.slice(0, 32)) const facts = [ { keys: blobProvider.toArchive(), @@ -67,6 +67,8 @@ export function blobAddProvider(context) { nb: { blob, exp: Number.MAX_SAFE_INTEGER, + // TODO: + // space }, expiration: Infinity, }) @@ -76,14 +78,14 @@ export function blobAddProvider(context) { ]) // Get receipt for `blob/allocate` if available, or schedule invocation if not - const allocatedGetRes = await allocationsStorage.get(space, blob.content) + const allocatedGetRes = await allocationsStorage.get(space, blob.digest) let blobAllocateReceipt /** @type {API.BlobAddress | undefined} */ let blobAllocateOutAddress // If already allocated, just get the allocate receipt // and the addresses if still pending to receive blob if (allocatedGetRes.ok) { - // TODO: Check expires + // TODO: Check expires? const receiptGet = await context.receiptsStorage.get(allocatefx.link()) if (receiptGet.error) { return receiptGet @@ -91,11 +93,11 @@ export function blobAddProvider(context) { blobAllocateReceipt = receiptGet.ok // Check if despite allocated, the blob is still not stored - const blobHasRes = await context.blobsStorage.has(blob.content) + const blobHasRes = await context.blobsStorage.has(blob.digest) if (blobHasRes.error) { return blobHasRes - // If still not stored, keep the allocate address to signal to the client - // that bytes MUST be sent through the `http/put` effect + // If still not stored, keep the allocate address to signal to the client + // that bytes MUST be sent through the `http/put` effect } else if (!blobHasRes.ok) { // @ts-expect-error receipt type is unknown blobAllocateOutAddress = blobAllocateReceipt.out.ok.address @@ -110,8 +112,8 @@ export function blobAddProvider(context) { error: new AwaitError({ cause: allocateRes.out.error, at: 'ucan/wait', - reference: ['.out.ok', allocatefx.cid] - }) + reference: ['.out.ok', allocatefx.cid], + }), } } // If this is a new allocation, `http/put` effect should be returned with address @@ -144,8 +146,8 @@ export function blobAddProvider(context) { // Create result object /** @type {API.OkBuilder} */ const result = Server.ok({ - location: { - 'ucan/await': ['.out.ok.claim', acceptfx.link()], + site: { + 'ucan/await': ['.out.ok.site', acceptfx.link()], }, }) @@ -159,7 +161,7 @@ export function blobAddProvider(context) { nb: { body: blob, url: blobAllocateOutAddress.url, - headers: blobAllocateOutAddress.headers + headers: blobAllocateOutAddress.headers, }, facts, expiration: Infinity, diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index ea349cf22..5a5990bf6 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -55,7 +55,7 @@ export function blobAllocateProvider(context) { // added to the space and there is no allocation change. // If record exists but is expired, it can be re-written if (allocationInsert.error.name === 'RecordKeyConflict') { - // TODO: Updates to new URL and expiration if expired? + // TODO: Should we return the same anyway and read the store to get address? return { ok: { size: 0 }, } @@ -68,7 +68,7 @@ export function blobAllocateProvider(context) { // Get presigned URL for the write target const expiresIn = 60 * 60 * 24 // 1 day const createUploadUrl = await context.blobsStorage.createUploadUrl( - blob.content, + blob.digest, blob.size, expiresIn ) @@ -79,7 +79,7 @@ export function blobAllocateProvider(context) { } // Check if blob already exists - const hasBlobStore = await context.blobsStorage.has(blob.content) + const hasBlobStore = await context.blobsStorage.has(blob.digest) if (hasBlobStore.error) { return hasBlobStore } @@ -100,7 +100,6 @@ export function blobAllocateProvider(context) { ok: { size: blob.size, address, - expiresAt: (new Date(Date.now() + expiresIn)).toISOString() }, } } diff --git a/packages/upload-api/src/blob/lib.js b/packages/upload-api/src/blob/lib.js index 663d93e68..55bf5ae20 100644 --- a/packages/upload-api/src/blob/lib.js +++ b/packages/upload-api/src/blob/lib.js @@ -64,14 +64,14 @@ export class AwaitError extends Failure { * @param {import('@ucanto/interface').Failure} source.cause - error that caused referenced `await` to fail */ constructor({ at, reference, cause }) { - super() - this.at = at - this.reference = reference - this.cause = cause + super() + this.at = at + this.reference = reference + this.cause = cause } describe() { - const [selector, task] = this.reference - return `Awaited (${selector} ${task}) reference at ${this.at} has failed:\n${this.cause}` + const [selector, task] = this.reference + return `Awaited (${selector} ${task}) reference at ${this.at} has failed:\n${this.cause}` } get name() { return AwaitErrorName diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index 405c4fe66..fcca50aef 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -5,16 +5,9 @@ import type { Failure, DID, } from '@ucanto/interface' -import { - BlobMultihash, - BlobListItem, -} from '@web3-storage/capabilities/types' +import { BlobMultihash, BlobListItem } from '@web3-storage/capabilities/types' -import { - RecordKeyConflict, - ListOptions, - ListResponse, -} from '../types.js' +import { RecordKeyConflict, ListOptions, ListResponse } from '../types.js' import { Storage } from './storage.js' export type TasksStorage = Storage @@ -39,7 +32,7 @@ export interface AllocationsStorage { } export interface Blob { - content: BlobMultihash + digest: BlobMultihash size: number } @@ -53,7 +46,7 @@ export interface BlobAddOutput extends Omit {} export interface BlobGetOutput { - blob: { content: Uint8Array; size: number } + blob: { digest: Uint8Array; size: number } invocation: UnknownLink } diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index 136a7cef7..872a6483f 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -73,6 +73,7 @@ export const ucanConcludeProvider = ({ // @ts-expect-error body exists in `http/put` but unknown type here blob: cap.nb.body, exp: Number.MAX_SAFE_INTEGER, + // TOOD: space }, expiration: Infinity, }) diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index bb465fea4..10ac0af99 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -29,7 +29,7 @@ export const test = { // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -45,7 +45,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -57,10 +57,10 @@ export const test = { } // Validate receipt - assert.ok(blobAdd.out.ok.location) - assert.equal(blobAdd.out.ok.location['ucan/await'][0], '.out.ok.claim') + assert.ok(blobAdd.out.ok.site) + assert.equal(blobAdd.out.ok.site['ucan/await'][0], '.out.ok.site') assert.ok( - blobAdd.out.ok.location['ucan/await'][1].equals(blobAdd.fx.join?.link()) + blobAdd.out.ok.site['ucan/await'][1].equals(blobAdd.fx.join?.link()) ) assert.ok(blobAdd.fx.join) @@ -122,7 +122,7 @@ export const test = { // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -138,7 +138,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -245,7 +245,7 @@ export const test = { // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes // create service connection const connection = connect({ @@ -260,7 +260,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size: Number.MAX_SAFE_INTEGER, }, }, @@ -282,8 +282,7 @@ export const test = { // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes - const digest = multihash.digest + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -299,7 +298,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -313,7 +312,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, cause: (await blobAddInvocation.delegate()).cid, @@ -337,7 +336,7 @@ export const test = { ) assert.deepEqual( blobAllocate.out.ok.address?.headers?.['x-amz-checksum-sha256'], - base64pad.baseEncode(digest) + base64pad.baseEncode(multihash.digest) ) const url = @@ -362,7 +361,7 @@ export const test = { if (!allocatedEntry) { throw new Error('Expected presigned allocatedEntry in response') } - assert.ok(equals(allocatedEntry.blob.content, content)) + assert.ok(equals(allocatedEntry.blob.digest, digest)) assert.equal(allocatedEntry.blob.size, size) // Validate presigned url usage @@ -381,7 +380,7 @@ export const test = { // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -397,7 +396,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -411,7 +410,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, cause: (await blobAddInvocation.delegate()).cid, @@ -447,7 +446,7 @@ export const test = { // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -463,7 +462,7 @@ export const test = { with: aliceSpaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -475,7 +474,7 @@ export const test = { with: bobSpaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -489,7 +488,7 @@ export const test = { with: aliceSpaceDid, nb: { blob: { - content, + digest, size, }, cause: (await aliceBlobAddInvocation.delegate()).cid, @@ -530,7 +529,7 @@ export const test = { with: bobSpaceDid, nb: { blob: { - content, + digest, size, }, cause: (await bobBlobAddInvocation.delegate()).cid, @@ -567,7 +566,7 @@ export const test = { const data = new Uint8Array([11, 22, 34, 44, 55]) const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -583,7 +582,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -597,7 +596,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, cause: (await blobAddInvocation.delegate()).cid, @@ -644,7 +643,7 @@ export const test = { const data = new Uint8Array([11, 22, 34, 44, 55]) const other = new Uint8Array([10, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -660,7 +659,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -674,7 +673,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, cause: (await blobAddInvocation.delegate()).cid, @@ -719,7 +718,7 @@ export const test = { // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -735,7 +734,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -749,7 +748,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, cause: (await blobAddInvocation.delegate()).cid, @@ -783,7 +782,7 @@ export const test = { // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) - const content = multihash.bytes + const digest = multihash.bytes const size = data.byteLength // create service connection @@ -810,7 +809,7 @@ export const test = { with: spaceDid, nb: { blob: { - content, + digest, size, }, }, @@ -879,11 +878,11 @@ export const test = { with: blobProvider.toDIDKey(), nb: { body: { - content, + digest, size, }, url: address.url, - headers: address.headers + headers: address.headers, }, facts: putfx.facts, expiration: Infinity, @@ -924,11 +923,9 @@ export const test = { // verify accept was scheduled const blobAcceptInvocation = await taskScheduled.promise assert.ok(blobAcceptInvocation) - assert.equal(blobAdd.out.ok.location['ucan/await'][0], '.out.ok.claim') + assert.equal(blobAdd.out.ok.site['ucan/await'][0], '.out.ok.site') assert.ok( - blobAdd.out.ok.location['ucan/await'][1].equals( - blobAcceptInvocation.cid - ) + blobAdd.out.ok.site['ucan/await'][1].equals(blobAcceptInvocation.cid) ) assert.ok(blobAdd.fx.join?.link().equals(blobAcceptInvocation.cid)) }, diff --git a/packages/upload-api/test/storage/allocations-storage.js b/packages/upload-api/test/storage/allocations-storage.js index 12810675a..115b5b3b8 100644 --- a/packages/upload-api/test/storage/allocations-storage.js +++ b/packages/upload-api/test/storage/allocations-storage.js @@ -17,7 +17,7 @@ export class AllocationsStorage { async insert({ space, invocation, ...output }) { if ( this.items.some( - (i) => i.space === space && equals(i.blob.content, output.blob.content) + (i) => i.space === space && equals(i.blob.digest, output.blob.digest) ) ) { return { @@ -40,7 +40,7 @@ export class AllocationsStorage { */ async get(space, blobMultihash) { const item = this.items.find( - (i) => i.space === space && equals(i.blob.content, blobMultihash) + (i) => i.space === space && equals(i.blob.digest, blobMultihash) ) if (!item) { return { error: { name: 'RecordNotFound', message: 'record not found' } } @@ -55,7 +55,7 @@ export class AllocationsStorage { */ async exists(space, blobMultihash) { const item = this.items.find( - (i) => i.space === space && equals(i.blob.content, blobMultihash) + (i) => i.space === space && equals(i.blob.digest, blobMultihash) ) return { ok: !!item } } diff --git a/packages/upload-api/test/storage/blobs-storage.js b/packages/upload-api/test/storage/blobs-storage.js index cf821c918..b5fa1e61f 100644 --- a/packages/upload-api/test/storage/blobs-storage.js +++ b/packages/upload-api/test/storage/blobs-storage.js @@ -127,8 +127,7 @@ export class BlobsStorage { * @param {number} expiresIn */ async createUploadUrl(multihash, size, expiresIn) { - const { bucket, accessKeyId, secretAccessKey, region, baseURL } = - this + const { bucket, accessKeyId, secretAccessKey, region, baseURL } = this const encodedMultihash = base58btc.encode(multihash) const multihashDigest = digestDecode(multihash) // sigv4 From 5a49c727da2073c561bfe38882f552a01ef0778e Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 9 Apr 2024 14:04:23 +0200 Subject: [PATCH 12/27] chore: address review --- packages/capabilities/src/ucan.js | 6 +++--- packages/capabilities/src/web3.storage/blob.js | 1 + packages/upload-api/src/blob/add.js | 5 ++++- packages/upload-api/src/blob/allocate.js | 1 - packages/upload-api/src/ucan/conclude.js | 2 +- packages/upload-api/test/handlers/blob.js | 8 ++++---- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/capabilities/src/ucan.js b/packages/capabilities/src/ucan.js index 51a07ffe0..b8ceb9d87 100644 --- a/packages/capabilities/src/ucan.js +++ b/packages/capabilities/src/ucan.js @@ -91,14 +91,14 @@ export const conclude = capability({ with: Schema.did(), nb: Schema.struct({ /** - * CID of the content with the UCANTO Message. + * CID of the content with the Receipt. */ - message: Schema.link(), + receipt: Schema.link(), }), derives: (claim, from) => // With field MUST be the same and(equalWith(claim, from)) || - and(checkLink(claim.nb.message, from.nb.message, 'nb.message')) || + and(checkLink(claim.nb.receipt, from.nb.receipt, 'nb.receipt')) || ok({}), }) diff --git a/packages/capabilities/src/web3.storage/blob.js b/packages/capabilities/src/web3.storage/blob.js index 4817f033e..80bfd41a1 100644 --- a/packages/capabilities/src/web3.storage/blob.js +++ b/packages/capabilities/src/web3.storage/blob.js @@ -88,6 +88,7 @@ export const accept = capability({ // TODO: space // space: SpaceDID, // TODO: _put? + // TODO: _allocate? }), derives: (claim, from) => { const result = equalBlob(claim, from) diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index ff8c9613b..cc425ed98 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -69,6 +69,8 @@ export function blobAddProvider(context) { exp: Number.MAX_SAFE_INTEGER, // TODO: // space + // TODO: awaits + //_put: { "ucan/await", [".out.ok", blobPut.link()] }, }, expiration: Infinity, }) @@ -122,6 +124,7 @@ export function blobAddProvider(context) { } // Create `blob/allocate` receipt invocation to inline as effect + // TODO: needs ucanto to accept any block const message = await Message.build({ receipts: [blobAllocateReceipt] }) const messageCar = await CAR.outbound.encode(message) const bytes = new Uint8Array(messageCar.body) @@ -133,7 +136,7 @@ export function blobAddProvider(context) { audience: id, with: id.toDIDKey(), nb: { - message: messageLink, + receipt: messageLink, }, expiration: Infinity, }) diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index 5a5990bf6..a60fee076 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -55,7 +55,6 @@ export function blobAllocateProvider(context) { // added to the space and there is no allocation change. // If record exists but is expired, it can be re-written if (allocationInsert.error.name === 'RecordKeyConflict') { - // TODO: Should we return the same anyway and read the store to get address? return { ok: { size: 0 }, } diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index 872a6483f..2d03e7b4c 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -19,7 +19,7 @@ export const ucanConcludeProvider = ({ }) => provide(conclude, async ({ capability, invocation }) => { const getBlockRes = await findBlock( - capability.nb.message, + capability.nb.receipt, invocation.iterateIPLDBlocks() ) if (getBlockRes.error) { diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 10ac0af99..48993310a 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -94,7 +94,7 @@ export const test = { // validate that scheduled allocate task executed and has its receipt content const getBlockRes = await findBlock( // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.message, + allocateUcanConcludefx.capabilities[0].nb.receipt, allocateUcanConcludefx.iterateIPLDBlocks() ) if (getBlockRes.error) { @@ -172,7 +172,7 @@ export const test = { } const getBlockRes = await findBlock( // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.message, + allocateUcanConcludefx.capabilities[0].nb.receipt, allocateUcanConcludefx.iterateIPLDBlocks() ) if (getBlockRes.error) { @@ -837,7 +837,7 @@ export const test = { } const getBlockRes = await findBlock( // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.message, + allocateUcanConcludefx.capabilities[0].nb.receipt, allocateUcanConcludefx.iterateIPLDBlocks() ) if (getBlockRes.error) { @@ -907,7 +907,7 @@ export const test = { audience: context.id, with: alice.did(), nb: { - message: messageLink, + receipt: messageLink, }, expiration: Infinity, }) From 6579dde3f2193aa81250701fd7aa4c6e1dc06aa9 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 9 Apr 2024 14:18:03 +0200 Subject: [PATCH 13/27] fix: derive blob provider from last 32 bytes from content --- packages/upload-api/src/blob/add.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index cc425ed98..379c4a486 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -41,7 +41,7 @@ export function blobAddProvider(context) { // of the `http/put` invocation. That way anyone with blob digest // could perform the invocation and issue receipt by deriving same // principal - const blobProvider = await ed25519.derive(blob.digest.slice(0, 32)) + const blobProvider = await ed25519.derive(blob.digest.slice(blob.digest.length - 32)) const facts = [ { keys: blobProvider.toArchive(), From 7618768dc2e3ae8df9b5f4983bdb639e0e772eae Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 9 Apr 2024 14:24:51 +0200 Subject: [PATCH 14/27] fix: try to fix tests --- .github/workflows/w3up-client.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/w3up-client.yml b/.github/workflows/w3up-client.yml index cd2a01196..a469b0f98 100644 --- a/.github/workflows/w3up-client.yml +++ b/.github/workflows/w3up-client.yml @@ -39,8 +39,7 @@ jobs: node-version: ${{ matrix.node_version }} registry-url: https://registry.npmjs.org/ cache: 'pnpm' - - run: pnpm --filter '@web3-storage/w3up-client...' install - - run: pnpm --filter '@web3-storage/w3up-client' attw + - run: pnpm install - uses: ./packages/w3up-client/.github/actions/test with: w3up-client-dir: ./packages/w3up-client/ From 4de3b9618d8403c3692f33c92fdab9deb7bbae1c Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 9 Apr 2024 16:27:02 +0200 Subject: [PATCH 15/27] chore: address review --- packages/capabilities/src/blob.js | 2 +- packages/capabilities/src/http.js | 2 +- packages/capabilities/src/types.ts | 15 ++++++++------- packages/capabilities/src/web3.storage/blob.js | 2 +- packages/upload-api/src/blob/add.js | 4 +++- packages/upload-api/src/ucan/conclude.js | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 4ef350b0d..8b42eed58 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -71,6 +71,6 @@ export const add = capability({ derives: equalBlob, }) -// ⚠️ We export imports here so they are not omitted in generated typedes +// ⚠️ We export imports here so they are not omitted in generated typedefs // @see https://github.com/microsoft/TypeScript/issues/51548 export { Schema } diff --git a/packages/capabilities/src/http.js b/packages/capabilities/src/http.js index 5f8d5d8c3..b53220c25 100644 --- a/packages/capabilities/src/http.js +++ b/packages/capabilities/src/http.js @@ -26,7 +26,7 @@ export const put = capability({ with: SpaceDID, nb: Schema.struct({ /** - * BodyBlob to allocate on the space. + * Description of body to send (digest/size). */ body: content, // TODO: what should be used? diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index cd18c02bf..20936f7cc 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -492,16 +492,17 @@ export interface BlobAddress { headers: Record } -export interface BlobItemNotFound extends Ucanto.Failure { - name: 'BlobItemNotFound' +export interface BlobNotFound extends Ucanto.Failure { + name: 'BlobNotFound' } +// If space has not enough space to allocate the blob. export interface BlobNotAllocableToSpace extends Ucanto.Failure { name: 'BlobNotAllocableToSpace' } export type BlobAllocateFailure = - | BlobItemNotFound + | BlobNotFound | BlobNotAllocableToSpace | Ucanto.Failure @@ -511,7 +512,7 @@ export interface BlobAcceptSuccess { } // TODO: We should type the store errors and add them here, instead of Ucanto.Failure -export type BlobAcceptFailure = BlobItemNotFound | Ucanto.Failure +export type BlobAcceptFailure = BlobNotFound | Ucanto.Failure // Store export type Store = InferInvokedCapability @@ -658,11 +659,11 @@ export type UCANRevokeFailure = /** * Error is raised when receipt is received for unknown invocation */ -export interface InvocationNotFoundForReceipt extends Ucanto.Failure { - name: 'InvocationNotFoundForReceipt' +export interface ReceiptInvocationNotFound extends Ucanto.Failure { + name: 'ReceiptInvocationNotFound' } -export type UCANConcludeFailure = InvocationNotFoundForReceipt | Ucanto.Failure +export type UCANConcludeFailure = ReceiptInvocationNotFound | Ucanto.Failure // Admin export type Admin = InferInvokedCapability diff --git a/packages/capabilities/src/web3.storage/blob.js b/packages/capabilities/src/web3.storage/blob.js index 80bfd41a1..912563d8b 100644 --- a/packages/capabilities/src/web3.storage/blob.js +++ b/packages/capabilities/src/web3.storage/blob.js @@ -104,6 +104,6 @@ export const accept = capability({ }, }) -// ⚠️ We export imports here so they are not omitted in generated typedes +// ⚠️ We export imports here so they are not omitted in generated typedefs // @see https://github.com/microsoft/TypeScript/issues/51548 export { Schema, Link } diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 379c4a486..cf277a22c 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -41,7 +41,9 @@ export function blobAddProvider(context) { // of the `http/put` invocation. That way anyone with blob digest // could perform the invocation and issue receipt by deriving same // principal - const blobProvider = await ed25519.derive(blob.digest.slice(blob.digest.length - 32)) + const blobProvider = await ed25519.derive( + blob.digest.slice(blob.digest.length - 32) + ) const facts = [ { keys: blobProvider.toArchive(), diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index 2d03e7b4c..104d7471f 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -39,7 +39,7 @@ export const ucanConcludeProvider = ({ throw new Error('receipt should exist') } - // TODO: Verify invocation exists failing with InvocationNotFoundForReceipt + // TODO: Verify invocation exists failing with ReceiptInvocationNotFound // Store receipt const receiptPutRes = await receiptsStorage.put(receipt) From e58e864757a03266e98590b286c911ba78e10711 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 9 Apr 2024 17:51:51 +0200 Subject: [PATCH 16/27] chore: clean up --- packages/capabilities/src/types.ts | 44 ++++++++--- packages/upload-api/src/blob/accept.js | 9 ++- packages/upload-api/src/blob/add.js | 10 ++- packages/upload-api/src/blob/allocate.js | 6 +- packages/upload-api/src/blob/lib.js | 7 +- packages/upload-api/src/types/storage.ts | 22 +----- packages/upload-api/src/ucan/conclude.js | 74 ++++++++++++------- packages/upload-api/test/handlers/blob.js | 4 +- .../test/storage/receipts-storage.js | 4 +- .../upload-api/test/storage/tasks-storage.js | 4 +- 10 files changed, 106 insertions(+), 78 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 20936f7cc..52af012fb 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -465,16 +465,20 @@ export interface BlobAddSuccess { } } -export interface BlobExceedsSizeLimit extends Ucanto.Failure { - name: 'BlobExceedsSizeLimit' +export interface BlobSizeOutsideOfSupportedRange extends Ucanto.Failure { + name: 'BlobSizeOutsideOfSupportedRange' } export interface AwaitError extends Ucanto.Failure { name: 'AwaitError' } -// TODO: We should type the store errors and add them here, instead of Ucanto.Failure -export type BlobAddFailure = BlobExceedsSizeLimit | AwaitError | Ucanto.Failure +// TODO: We need Ucanto.Failure because provideAdvanced can't handle errors without it +export type BlobAddFailure = + | BlobSizeOutsideOfSupportedRange + | AwaitError + | StorageGetError + | Ucanto.Failure export interface BlobListItem { blob: BlobModel @@ -492,28 +496,44 @@ export interface BlobAddress { headers: Record } -export interface BlobNotFound extends Ucanto.Failure { - name: 'BlobNotFound' -} - // If space has not enough space to allocate the blob. export interface BlobNotAllocableToSpace extends Ucanto.Failure { name: 'BlobNotAllocableToSpace' } -export type BlobAllocateFailure = - | BlobNotFound - | BlobNotAllocableToSpace - | Ucanto.Failure +export type BlobAllocateFailure = BlobNotAllocableToSpace | Ucanto.Failure // Blob accept export interface BlobAcceptSuccess { site: Link } +export interface BlobNotFound extends Ucanto.Failure { + name: 'BlobNotFound' +} + // TODO: We should type the store errors and add them here, instead of Ucanto.Failure export type BlobAcceptFailure = BlobNotFound | Ucanto.Failure +// Storage errors +export type StoragePutError = StorageOperationError | EncodeRecordFailed +export type StorageGetError = + | StorageOperationError + | EncodeRecordFailed + | RecordNotFound + +export interface StorageOperationError extends Error { + name: 'StorageOperationFailed' +} + +export interface RecordNotFound extends Error { + name: 'RecordNotFound' +} + +export interface EncodeRecordFailed extends Error { + name: 'EncodeRecordFailed' +} + // Store export type Store = InferInvokedCapability export type StoreAdd = InferInvokedCapability diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index c35924d25..4c4afec28 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -8,6 +8,9 @@ import { CAR } from '@ucanto/core' import * as API from '../types.js' import { BlobItemNotFound } from './lib.js' +const R2_REGION = 'auto' +const R2_BUCKET = 'carpark-prod-0' + /** * @param {API.W3ServiceContext} context * @returns {API.ServiceMethod} @@ -28,14 +31,12 @@ export function blobAcceptProvider(context) { // TODO: we need to support multihash in claims, or specify hardcoded codec const digest = new Digest(sha256.code, 32, blob.digest, blob.digest) const content = createLink(CAR.code, digest) - const w3link = `https://w3s.link/ipfs/${content.toString()}` + const w3link = `https://w3s.link/ipfs/${content.toString()}?origin=r2://${R2_REGION}/${R2_BUCKET}` - // TODO: Set bucket name - // TODO: return content commitment const locationClaim = await Assert.location .invoke({ issuer: context.id, - // TODO: we need user agent DID + // TODO: we need space CID here audience: context.id, with: context.id.toDIDKey(), nb: { diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index cf277a22c..8aa747f7e 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -8,7 +8,7 @@ import * as HTTP from '@web3-storage/capabilities/http' import * as UCAN from '@web3-storage/capabilities/ucan' import * as API from '../types.js' -import { BlobExceedsSizeLimit, AwaitError } from './lib.js' +import { BlobSizeOutsideOfSupportedRange, AwaitError } from './lib.js' /** * @param {API.BlobServiceContext} context @@ -33,7 +33,7 @@ export function blobAddProvider(context) { // Verify blob is within accept size if (blob.size > maxUploadSize) { return { - error: new BlobExceedsSizeLimit(maxUploadSize), + error: new BlobSizeOutsideOfSupportedRange(maxUploadSize), } } @@ -89,10 +89,12 @@ export function blobAddProvider(context) { // If already allocated, just get the allocate receipt // and the addresses if still pending to receive blob if (allocatedGetRes.ok) { - // TODO: Check expires? + // TODO: How to check expired? const receiptGet = await context.receiptsStorage.get(allocatefx.link()) if (receiptGet.error) { - return receiptGet + return { + error: receiptGet.error, + } } blobAllocateReceipt = receiptGet.ok diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index a60fee076..7ea2b985a 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -60,7 +60,7 @@ export function blobAllocateProvider(context) { } } return { - error: new Server.Failure('failed to allocate blob bytes'), + error: allocationInsert.error, } } @@ -72,9 +72,7 @@ export function blobAllocateProvider(context) { expiresIn ) if (createUploadUrl.error) { - return { - error: new Server.Failure('failed to provide presigned url'), - } + return createUploadUrl } // Check if blob already exists diff --git a/packages/upload-api/src/blob/lib.js b/packages/upload-api/src/blob/lib.js index 55bf5ae20..bbdaa621d 100644 --- a/packages/upload-api/src/blob/lib.js +++ b/packages/upload-api/src/blob/lib.js @@ -29,8 +29,9 @@ export class BlobItemNotFound extends Failure { } } -export const BlobExceedsSizeLimitName = 'BlobExceedsSizeLimit' -export class BlobExceedsSizeLimit extends Failure { +export const BlobSizeOutsideOfSupportedRangeName = + 'BlobSizeOutsideOfSupportedRange' +export class BlobSizeOutsideOfSupportedRange extends Failure { /** * @param {Number} maxUploadSize */ @@ -40,7 +41,7 @@ export class BlobExceedsSizeLimit extends Failure { } get name() { - return BlobExceedsSizeLimitName + return BlobSizeOutsideOfSupportedRangeName } describe() { diff --git a/packages/upload-api/src/types/storage.ts b/packages/upload-api/src/types/storage.ts index bc3b18ff2..7cf5fea72 100644 --- a/packages/upload-api/src/types/storage.ts +++ b/packages/upload-api/src/types/storage.ts @@ -1,4 +1,8 @@ import type { Unit, Result } from '@ucanto/interface' +import { + StorageGetError, + StoragePutError, +} from '@web3-storage/capabilities/types' export interface Storage { /** @@ -14,21 +18,3 @@ export interface Storage { */ has: (key: RecKey) => Promise> } - -export type StoragePutError = StorageOperationError | EncodeRecordFailed -export type StorageGetError = - | StorageOperationError - | EncodeRecordFailed - | RecordNotFound - -export interface StorageOperationError extends Error { - name: 'StorageOperationFailed' -} - -export interface RecordNotFound extends Error { - name: 'RecordNotFound' -} - -export interface EncodeRecordFailed extends Error { - name: 'EncodeRecordFailed' -} diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index 104d7471f..f66a676ac 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -18,31 +18,23 @@ export const ucanConcludeProvider = ({ tasksScheduler, }) => provide(conclude, async ({ capability, invocation }) => { - const getBlockRes = await findBlock( + const receiptGet = await getReceipt( capability.nb.receipt, invocation.iterateIPLDBlocks() ) - if (getBlockRes.error) { - return getBlockRes + if (receiptGet.error) { + return receiptGet } - const messageCar = CAR.codec.decode(getBlockRes.ok) - const message = Message.view({ - root: messageCar.roots[0].cid, - store: messageCar.blocks, - }) - // TODO: check number of receipts - const receiptKey = Array.from(message.receipts.keys())[0] - const receipt = message.receipts.get(receiptKey) - - if (!receipt) { - throw new Error('receipt should exist') + // Verify invocation exists failing with ReceiptInvocationNotFound + const ranInvocation = receiptGet.ok.ran + const httpPutTaskGetRes = await tasksStorage.get(ranInvocation.link()) + if (httpPutTaskGetRes.error) { + return httpPutTaskGetRes } - // TODO: Verify invocation exists failing with ReceiptInvocationNotFound - // Store receipt - const receiptPutRes = await receiptsStorage.put(receipt) + const receiptPutRes = await receiptsStorage.put(receiptGet.ok) if (receiptPutRes.error) { return { error: receiptPutRes.error, @@ -50,15 +42,6 @@ export const ucanConcludeProvider = ({ } // THIS IS A TEMPORARY HACK - // Schedule `blob/accept` - const ranInvocation = receipt.ran - - // Get invocation - const httpPutTaskGetRes = await tasksStorage.get(ranInvocation.link()) - if (httpPutTaskGetRes.error) { - return httpPutTaskGetRes - } - // Schedule `blob/accept` if there is a `http/put` capability const scheduleRes = await Promise.all( httpPutTaskGetRes.ok.capabilities @@ -73,7 +56,7 @@ export const ucanConcludeProvider = ({ // @ts-expect-error body exists in `http/put` but unknown type here blob: cap.nb.body, exp: Number.MAX_SAFE_INTEGER, - // TOOD: space + // TODO: space }, expiration: Infinity, }) @@ -95,6 +78,43 @@ export const ucanConcludeProvider = ({ } }) +/** + * @param {import('multiformats').UnknownLink} receiptLink + * @param {IterableIterator>} blocks + */ +export const getReceipt = async (receiptLink, blocks) => { + const getBlockRes = await findBlock(receiptLink, blocks) + if (getBlockRes.error) { + return getBlockRes + } + + const messageCar = CAR.codec.decode(getBlockRes.ok) + const message = Message.view({ + root: messageCar.roots[0].cid, + store: messageCar.blocks, + }) + + const receiptKeys = Array.from(message.receipts.keys()) + if (receiptKeys.length !== 1) { + return { + error: { + name: 'UnexpectedNumberOfReceipts', + message: `${receiptKeys.length} receipts received`, + }, + } + } + + const receiptId = receiptKeys[0] + const receipt = message.receipts.get(receiptId) + if (!receipt) { + throw new Error('receipt must exist given a key exists for it') + } + + return { + ok: receipt + } +} + /** * @param {import('multiformats').UnknownLink} cid * @param {IterableIterator>} blocks diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 48993310a..d89e06db6 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -15,7 +15,7 @@ import { base64pad } from 'multiformats/bases/base64' import { provisionProvider } from '../helpers/utils.js' import { createServer, connect } from '../../src/lib.js' import { alice, bob, createSpace, registerSpace } from '../util.js' -import { BlobExceedsSizeLimitName } from '../../src/blob/lib.js' +import { BlobSizeOutsideOfSupportedRangeName } from '../../src/blob/lib.js' import { findBlock } from '../../src/ucan/conclude.js' /** @@ -271,7 +271,7 @@ export const test = { throw new Error('invocation should have failed') } assert.ok(blobAdd.out.error, 'invocation should have failed') - assert.equal(blobAdd.out.error.name, BlobExceedsSizeLimitName) + assert.equal(blobAdd.out.error.name, BlobSizeOutsideOfSupportedRangeName) }, 'blob/allocate allocates to space and returns presigned url': async ( assert, diff --git a/packages/upload-api/test/storage/receipts-storage.js b/packages/upload-api/test/storage/receipts-storage.js index 914fef611..4faf5b365 100644 --- a/packages/upload-api/test/storage/receipts-storage.js +++ b/packages/upload-api/test/storage/receipts-storage.js @@ -3,8 +3,8 @@ import * as API from '../../src/types.js' import { RecordNotFound } from '../../src/errors.js' /** - * @typedef {import('../../src/types/storage.js').StorageGetError} StorageGetError - * @typedef {import('../../src/types/storage.js').StoragePutError} StoragePutError + * @typedef {import('@web3-storage/capabilities/types').StorageGetError} StorageGetError + * @typedef {import('@web3-storage/capabilities/types').StoragePutError} StoragePutError * @typedef {import('@ucanto/interface').UnknownLink} UnknownLink * @typedef {import('@ucanto/interface').Receipt} Receipt */ diff --git a/packages/upload-api/test/storage/tasks-storage.js b/packages/upload-api/test/storage/tasks-storage.js index 43468a8bb..7a704e6a2 100644 --- a/packages/upload-api/test/storage/tasks-storage.js +++ b/packages/upload-api/test/storage/tasks-storage.js @@ -3,8 +3,8 @@ import * as API from '../../src/types.js' import { RecordNotFound } from '../../src/errors.js' /** - * @typedef {import('../../src/types/storage.js').StorageGetError} StorageGetError - * @typedef {import('../../src/types/storage.js').StoragePutError} StoragePutError + * @typedef {import('@web3-storage/capabilities/types').StorageGetError} StorageGetError + * @typedef {import('@web3-storage/capabilities/types').StoragePutError} StoragePutError * @typedef {import('@ucanto/interface').UnknownLink} UnknownLink * @typedef {import('@ucanto/interface').Invocation} Invocation */ From 5adcb0db74691c7dbe4cfa2081ca7b51901a7af9 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 10 Apr 2024 14:31:53 +0200 Subject: [PATCH 17/27] fix: adopt fully ucan await pipeline --- packages/capabilities/src/http.js | 19 +- packages/capabilities/src/utils.js | 6 +- .../capabilities/src/web3.storage/blob.js | 29 +- packages/upload-api/src/blob/add.js | 476 +++++++++++------- packages/upload-api/src/ucan/conclude.js | 106 ++-- packages/upload-api/test/handlers/blob.js | 154 +++--- 6 files changed, 437 insertions(+), 353 deletions(-) diff --git a/packages/capabilities/src/http.js b/packages/capabilities/src/http.js index b53220c25..a240f6aa0 100644 --- a/packages/capabilities/src/http.js +++ b/packages/capabilities/src/http.js @@ -10,7 +10,7 @@ */ import { capability, Schema, ok } from '@ucanto/validator' import { content } from './blob.js' -import { equal, equalBody, equalWith, SpaceDID, and } from './utils.js' +import { equal, equalBody, equalWith, SpaceDID, Await, and } from './utils.js' /** * `http/put` capability invocation MAY be performed by any agent on behalf of the subject. @@ -29,27 +29,14 @@ export const put = capability({ * Description of body to send (digest/size). */ body: content, - // TODO: what should be used? /** * HTTP(S) location that can receive blob content via HTTP PUT request. */ - // url: Schema.struct({ - // 'ucan/await': Schema.unknown(), - // }).optional(), - // /** - // * HTTP headers. - // */ - // headers: Schema.struct({ - // 'ucan/await': Schema.unknown(), - // }).optional(), - /** - * HTTP(S) location that can receive blob content via HTTP PUT request. - */ - url: Schema.string(), + url: Schema.string().or(Await), /** * HTTP headers. */ - headers: Schema.dictionary({ value: Schema.string() }), + headers: Schema.dictionary({ value: Schema.string() }).or(Await), }), derives: (claim, from) => { return ( diff --git a/packages/capabilities/src/utils.js b/packages/capabilities/src/utils.js index d15afd523..00e512d1c 100644 --- a/packages/capabilities/src/utils.js +++ b/packages/capabilities/src/utils.js @@ -1,4 +1,4 @@ -import { DID, fail, ok } from '@ucanto/validator' +import { DID, Schema, fail, ok } from '@ucanto/validator' // eslint-disable-next-line no-unused-vars import * as Types from '@ucanto/interface' @@ -11,6 +11,10 @@ export const SpaceDID = DID.match({ method: 'key' }) export const AccountDID = DID.match({ method: 'mailto' }) +export const Await = Schema.struct({ + 'ucan/await': Schema.tuple([Schema.string(), Schema.link()]), +}) + /** * Check URI can be delegated * diff --git a/packages/capabilities/src/web3.storage/blob.js b/packages/capabilities/src/web3.storage/blob.js index 912563d8b..601b37192 100644 --- a/packages/capabilities/src/web3.storage/blob.js +++ b/packages/capabilities/src/web3.storage/blob.js @@ -1,4 +1,4 @@ -import { capability, Schema, Link, ok, fail } from '@ucanto/validator' +import { capability, Schema, Link, ok } from '@ucanto/validator' import { content } from '../blob.js' import { equalBlob, @@ -7,6 +7,7 @@ import { and, equal, checkLink, + Await, } from '../utils.js' /** @@ -85,22 +86,20 @@ export const accept = capability({ /** * DID of the user space where allocation took place */ - // TODO: space - // space: SpaceDID, - // TODO: _put? - // TODO: _allocate? + space: SpaceDID, + /** + * This task is blocked on `http/put` receipt available + */ + _put: Await, }), derives: (claim, from) => { - const result = equalBlob(claim, from) - if (result.error) { - return result - } else if (claim.nb.exp !== undefined && from.nb.exp !== undefined) { - return claim.nb.exp > from.nb.exp - ? fail(`exp constraint violation: ${claim.nb.exp} > ${from.nb.exp}`) - : ok({}) - } else { - return ok({}) - } + return ( + and(equalWith(claim, from)) || + and(equalBlob(claim, from)) || + and(equal(claim.nb.exp, from.nb.exp, 'exp')) || + and(equal(claim.nb.space, from.nb.space, 'space')) || + ok({}) + ) }, }) diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 8aa747f7e..6275e6bba 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -1,13 +1,11 @@ import * as Server from '@ucanto/server' -import { Message } from '@ucanto/core' import { ed25519 } from '@ucanto/principal' -import { CAR } from '@ucanto/transport' import * as Blob from '@web3-storage/capabilities/blob' import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import * as HTTP from '@web3-storage/capabilities/http' -import * as UCAN from '@web3-storage/capabilities/ucan' import * as API from '../types.js' +import { createConcludeInvocation } from '../ucan/conclude.js' import { BlobSizeOutsideOfSupportedRange, AwaitError } from './lib.js' /** @@ -18,217 +16,345 @@ export function blobAddProvider(context) { return Server.provideAdvanced({ capability: Blob.add, handler: async ({ capability, invocation }) => { - const { - id, - allocationsStorage, - maxUploadSize, - getServiceConnection, - tasksStorage, - } = context + // Prepare context const { blob } = capability.nb const space = /** @type {import('@ucanto/interface').DIDKey} */ ( Server.DID.parse(capability.with).did() ) // Verify blob is within accept size - if (blob.size > maxUploadSize) { + if (blob.size > context.maxUploadSize) { return { - error: new BlobSizeOutsideOfSupportedRange(maxUploadSize), + error: new BlobSizeOutsideOfSupportedRange(context.maxUploadSize), } } - // We derive principal from the blob multihash to be an audience - // of the `http/put` invocation. That way anyone with blob digest - // could perform the invocation and issue receipt by deriving same - // principal - const blobProvider = await ed25519.derive( - blob.digest.slice(blob.digest.length - 32) - ) - const facts = [ - { - keys: blobProvider.toArchive(), - }, - ] - - // Create web3.storage/blob/* invocations - const blobAllocate = W3sBlob.allocate.invoke({ - issuer: id, - audience: id, - with: id.did(), - nb: { - blob, - cause: invocation.link(), - space, - }, - expiration: Infinity, + // Create next tasks + const next = await createNextTasks({ + context, + blob, + space, + cause: invocation.link(), }) - const blobAccept = W3sBlob.accept.invoke({ - issuer: id, - audience: id, - with: id.toDIDKey(), - nb: { - blob, - exp: Number.MAX_SAFE_INTEGER, - // TODO: - // space - // TODO: awaits - //_put: { "ucan/await", [".out.ok", blobPut.link()] }, - }, - expiration: Infinity, + + // Schedule allocate + const scheduleAllocateRes = await scheduleAllocate({ + context, + allocate: next.allocate, + allocatefx: next.allocatefx, }) - const [allocatefx, acceptfx] = await Promise.all([ - blobAllocate.delegate(), - blobAccept.delegate(), - ]) - - // Get receipt for `blob/allocate` if available, or schedule invocation if not - const allocatedGetRes = await allocationsStorage.get(space, blob.digest) - let blobAllocateReceipt - /** @type {API.BlobAddress | undefined} */ - let blobAllocateOutAddress - // If already allocated, just get the allocate receipt - // and the addresses if still pending to receive blob - if (allocatedGetRes.ok) { - // TODO: How to check expired? - const receiptGet = await context.receiptsStorage.get(allocatefx.link()) - if (receiptGet.error) { - return { - error: receiptGet.error, - } - } - blobAllocateReceipt = receiptGet.ok - - // Check if despite allocated, the blob is still not stored - const blobHasRes = await context.blobsStorage.has(blob.digest) - if (blobHasRes.error) { - return blobHasRes - // If still not stored, keep the allocate address to signal to the client - // that bytes MUST be sent through the `http/put` effect - } else if (!blobHasRes.ok) { - // @ts-expect-error receipt type is unknown - blobAllocateOutAddress = blobAllocateReceipt.out.ok.address - } - } - // if not already allocated, schedule `blob/allocate` - else { - // Execute allocate invocation - const allocateRes = await blobAllocate.execute(getServiceConnection()) - if (allocateRes.out.error) { - return { - error: new AwaitError({ - cause: allocateRes.out.error, - at: 'ucan/wait', - reference: ['.out.ok', allocatefx.cid], - }), - } - } - // If this is a new allocation, `http/put` effect should be returned with address - blobAllocateOutAddress = allocateRes.out.ok.address - blobAllocateReceipt = allocateRes + if (scheduleAllocateRes.error) { + return scheduleAllocateRes } - // Create `blob/allocate` receipt invocation to inline as effect - // TODO: needs ucanto to accept any block - const message = await Message.build({ receipts: [blobAllocateReceipt] }) - const messageCar = await CAR.outbound.encode(message) - const bytes = new Uint8Array(messageCar.body) - const messageLink = await CAR.codec.link(bytes) - - const allocateUcanConcludefx = await UCAN.conclude - .invoke({ - issuer: id, - audience: id, - with: id.toDIDKey(), - nb: { - receipt: messageLink, - }, - expiration: Infinity, - }) - .delegate() - allocateUcanConcludefx.attach({ - bytes, - cid: messageLink, + // Schedule put + const schedulePutRes = await schedulePut({ + context, + putfx: next.putfx, }) + if (schedulePutRes.error) { + return schedulePutRes + } // Create result object /** @type {API.OkBuilder} */ const result = Server.ok({ site: { - 'ucan/await': ['.out.ok.site', acceptfx.link()], + 'ucan/await': ['.out.ok.site', next.acceptfx.link()], }, }) - // In case blob allocate provided an address to write - // the blob is still not stored - if (blobAllocateOutAddress) { - const blobPut = HTTP.put.invoke({ - issuer: blobProvider, - audience: blobProvider, - with: blobProvider.toDIDKey(), - nb: { - body: blob, - url: blobAllocateOutAddress.url, - headers: blobAllocateOutAddress.headers, - }, - facts, - expiration: Infinity, - }) - - const putfx = await blobPut.delegate() - - // store `http/put` invocation - // TODO: store implementation - // const archiveDelegationRes = await putfx.archive() - // if (archiveDelegationRes.error) { - // return { - // error: archiveDelegationRes.error - // } - // } - const invocationPutRes = await tasksStorage.put(putfx) - if (invocationPutRes.error) { - return { - error: invocationPutRes.error, - } - } - + // In case there is no receipt for concludePutfx, we can return + if (!schedulePutRes.ok.concludePutfx) { return ( result // 1. System attempts to allocate memory in user space for the blob. - .fork(allocatefx) - .fork(allocateUcanConcludefx) + .fork(next.allocatefx) + .fork(scheduleAllocateRes.ok.concludeAllocatefx) // 2. System requests user agent (or anyone really) to upload the content // corresponding to the blob // via HTTP PUT to given location. - .fork(putfx) + .fork(next.putfx) // 3. System will attempt to accept uploaded content that matches blob // multihash and size. - .join(acceptfx) + .join(next.acceptfx) ) } - // Add allocate receipt if allocate was executed - if (allocateUcanConcludefx) { - return ( - result + // schedule accept if there is http/put receipt available + const scheduleAcceptRes = await scheduleAccept({ + context, + accept: next.accept, + acceptfx: next.acceptfx, + }) + if (scheduleAcceptRes.error) { + return scheduleAcceptRes + } + + return scheduleAcceptRes.ok.concludeAcceptfx + ? result // 1. System attempts to allocate memory in user space for the blob. - .fork(allocatefx) - .fork(allocateUcanConcludefx) + .fork(next.allocatefx) + .fork(scheduleAllocateRes.ok.concludeAllocatefx) + // 2. System requests user agent (or anyone really) to upload the content + // corresponding to the blob + // via HTTP PUT to given location. + .fork(next.putfx) + .fork(schedulePutRes.ok.concludePutfx) // 3. System will attempt to accept uploaded content that matches blob // multihash and size. - .join(acceptfx) - ) + .join(next.acceptfx) + .fork(scheduleAcceptRes.ok.concludeAcceptfx) + : result + // 1. System attempts to allocate memory in user space for the blob. + .fork(next.allocatefx) + .fork(scheduleAllocateRes.ok.concludeAllocatefx) + // 2. System requests user agent (or anyone really) to upload the content + // corresponding to the blob + // via HTTP PUT to given location. + .fork(next.putfx) + .fork(schedulePutRes.ok.concludePutfx) + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. + .join(next.acceptfx) + }, + }) +} + +/** + * Schedule Put task to be run by agent. + * A `http/put` task is stored by the service, if it does not exist + * and a receipt is fetched if already available. + * + * @param {object} scheduleAcceptProps + * @param {API.BlobServiceContext} scheduleAcceptProps.context + * @param {API.IssuedInvocationView} scheduleAcceptProps.accept + * @param {API.Invocation} scheduleAcceptProps.acceptfx + */ +async function scheduleAccept({ context, accept, acceptfx }) { + let blobAcceptReceipt + + // Get receipt for `blob/accept` if available, otherwise schedule invocation + const receiptGet = await context.receiptsStorage.get(acceptfx.link()) + if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { + return { + error: receiptGet.error, + } + } else if (receiptGet.ok) { + blobAcceptReceipt = receiptGet.ok + } + + // if not already accepted schedule `blob/accept` + if (!blobAcceptReceipt) { + // Execute accept invocation + const acceptRes = await accept.execute(context.getServiceConnection()) + if (acceptRes.out.error) { + return { + error: new AwaitError({ + cause: acceptRes.out.error, + at: 'ucan/wait', + reference: ['.out.ok', acceptfx.cid], + }), } + } + blobAcceptReceipt = acceptRes + } - // Blob was already allocated and is already stored in the system - return ( - result - // 1. System allocated memory in user space for the blob. - .fork(allocatefx) - .fork(allocateUcanConcludefx) - // 3. System will attempt to accept uploaded content that matches blob - // multihash and size. - .join(acceptfx) - ) + // Create `blob/accept` receipt as conclude invocation to inline as effect + const concludeAccept = createConcludeInvocation( + context.id, + context.id, + blobAcceptReceipt + ) + return { + ok: { + concludeAcceptfx: await concludeAccept.delegate(), }, + } +} + +/** + * Schedule Put task to be run by agent. + * A `http/put` task is stored by the service, if it does not exist + * and a receipt is fetched if already available. + * + * @param {object} schedulePutProps + * @param {API.BlobServiceContext} schedulePutProps.context + * @param {API.Invocation} schedulePutProps.putfx + */ +async function schedulePut({ context, putfx }) { + // Get receipt for `http/put` if available + const receiptGet = await context.receiptsStorage.get(putfx.link()) + if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { + return { + error: receiptGet.error, + } + } else if (receiptGet.ok) { + // Create `blob/allocate` receipt as conclude invocation to inline as effect + const concludePut = createConcludeInvocation( + context.id, + context.id, + receiptGet.ok + ) + return { + ok: { + concludePutfx: await concludePut.delegate(), + }, + } + } + + // store `http/put` invocation + const invocationPutRes = await context.tasksStorage.put(putfx) + if (invocationPutRes.error) { + // TODO: If already available, do not error? + return { + error: invocationPutRes.error, + } + } + + // TODO: store implementation + // const archiveDelegationRes = await putfx.archive() + // if (archiveDelegationRes.error) { + // return { + // error: archiveDelegationRes.error + // } + // } + + return { + ok: {}, + } +} + +/** + * Schedule allocate task to be run. + * If there is a non expired receipt, it is returned insted of runing the task again. + * Otherwise, allocation task is scheduled. + * + * @param {object} scheduleAllocateProps + * @param {API.BlobServiceContext} scheduleAllocateProps.context + * @param {API.IssuedInvocationView} scheduleAllocateProps.allocate + * @param {API.Invocation} scheduleAllocateProps.allocatefx + */ +async function scheduleAllocate({ context, allocate, allocatefx }) { + let blobAllocateReceipt + + // Get receipt for `blob/allocate` if available, otherwise schedule invocation + const receiptGet = await context.receiptsStorage.get(allocatefx.link()) + if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { + return { + error: receiptGet.error, + } + } else if (receiptGet.ok) { + // TODO: check expired by adding to receipt? + blobAllocateReceipt = receiptGet.ok + } + + // if not already allocated (or expired) schedule `blob/allocate` + if (!blobAllocateReceipt) { + // Execute allocate invocation + const allocateRes = await allocate.execute(context.getServiceConnection()) + if (allocateRes.out.error) { + return { + error: new AwaitError({ + cause: allocateRes.out.error, + at: 'ucan/wait', + reference: ['.out.ok', allocatefx.cid], + }), + } + } + blobAllocateReceipt = allocateRes + } + + // Create `blob/allocate` receipt as conclude invocation to inline as effect + const concludeAllocate = createConcludeInvocation( + context.id, + context.id, + blobAllocateReceipt + ) + return { + ok: { + concludeAllocatefx: await concludeAllocate.delegate(), + }, + } +} + +/** + * Create `blob/add` next tasks. + * + * @param {object} nextProps + * @param {API.BlobServiceContext} nextProps.context + * @param {API.BlobModel} nextProps.blob + * @param {API.DIDKey} nextProps.space + * @param {API.Link} nextProps.cause + */ +async function createNextTasks({ context, blob, space, cause }) { + // 1. Create web3.storage/blob/allocate invocation and task + const allocate = W3sBlob.allocate.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + blob, + cause: cause, + space, + }, + expiration: Infinity, + }) + const allocatefx = await allocate.delegate() + + // 2. Create http/put invocation ans task + + // We derive principal from the blob multihash to be an audience + // of the `http/put` invocation. That way anyone with blob digest + // could perform the invocation and issue receipt by deriving same + // principal + const blobProvider = await ed25519.derive( + blob.digest.slice(blob.digest.length - 32) + ) + const facts = [ + { + keys: blobProvider.toArchive(), + }, + ] + const put = HTTP.put.invoke({ + issuer: blobProvider, + audience: blobProvider, + with: blobProvider.toDIDKey(), + nb: { + body: blob, + url: { + 'ucan/await': ['.out.ok.address.url', allocatefx.cid], + }, + headers: { + 'ucan/await': ['.out.ok.address.headers', allocatefx.cid], + }, + }, + facts, + expiration: Infinity, }) + const putfx = await put.delegate() + + // 3. Create web3.storage/blob/accept invocation and task + const accept = W3sBlob.accept.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.toDIDKey(), + nb: { + blob, + exp: Number.MAX_SAFE_INTEGER, + space, + _put: { 'ucan/await': ['.out.ok', putfx.link()] }, + }, + expiration: Infinity, + }) + const acceptfx = await accept.delegate() + + return { + allocate, + allocatefx, + put, + putfx, + accept, + acceptfx, + } } diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index f66a676ac..66015dabe 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -1,6 +1,5 @@ import { provide } from '@ucanto/server' -import { Message } from '@ucanto/core' -import { CAR } from '@ucanto/transport' +import { Receipt } from '@ucanto/core' import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import * as HTTP from '@web3-storage/capabilities/http' import { conclude } from '@web3-storage/capabilities/ucan' @@ -17,24 +16,18 @@ export const ucanConcludeProvider = ({ tasksStorage, tasksScheduler, }) => - provide(conclude, async ({ capability, invocation }) => { - const receiptGet = await getReceipt( - capability.nb.receipt, - invocation.iterateIPLDBlocks() - ) - if (receiptGet.error) { - return receiptGet - } + provide(conclude, async ({ invocation }) => { + const receipt = getConcludeReceipt(invocation) // Verify invocation exists failing with ReceiptInvocationNotFound - const ranInvocation = receiptGet.ok.ran + const ranInvocation = receipt.ran const httpPutTaskGetRes = await tasksStorage.get(ranInvocation.link()) if (httpPutTaskGetRes.error) { return httpPutTaskGetRes } // Store receipt - const receiptPutRes = await receiptsStorage.put(receiptGet.ok) + const receiptPutRes = await receiptsStorage.put(receipt) if (receiptPutRes.error) { return { error: receiptPutRes.error, @@ -56,7 +49,11 @@ export const ucanConcludeProvider = ({ // @ts-expect-error body exists in `http/put` but unknown type here blob: cap.nb.body, exp: Number.MAX_SAFE_INTEGER, - // TODO: space + // TODO: corect space + space: id.toDIDKey(), + _put: { + 'ucan/await': ['.out.ok', ranInvocation.link()], + }, }, expiration: Infinity, }) @@ -78,43 +75,6 @@ export const ucanConcludeProvider = ({ } }) -/** - * @param {import('multiformats').UnknownLink} receiptLink - * @param {IterableIterator>} blocks - */ -export const getReceipt = async (receiptLink, blocks) => { - const getBlockRes = await findBlock(receiptLink, blocks) - if (getBlockRes.error) { - return getBlockRes - } - - const messageCar = CAR.codec.decode(getBlockRes.ok) - const message = Message.view({ - root: messageCar.roots[0].cid, - store: messageCar.blocks, - }) - - const receiptKeys = Array.from(message.receipts.keys()) - if (receiptKeys.length !== 1) { - return { - error: { - name: 'UnexpectedNumberOfReceipts', - message: `${receiptKeys.length} receipts received`, - }, - } - } - - const receiptId = receiptKeys[0] - const receipt = message.receipts.get(receiptId) - if (!receipt) { - throw new Error('receipt must exist given a key exists for it') - } - - return { - ok: receipt - } -} - /** * @param {import('multiformats').UnknownLink} cid * @param {IterableIterator>} blocks @@ -136,3 +96,49 @@ export const findBlock = async (cid, blocks) => { ok: bytes, } } + +/** + * @param {import('@ucanto/interface').Invocation} concludeFx + */ +export function getConcludeReceipt(concludeFx) { + const receiptBlocks = new Map() + for (const block of concludeFx.iterateIPLDBlocks()) { + receiptBlocks.set(`${block.cid}`, block) + } + return Receipt.view({ + // @ts-expect-error object of type unknown + root: concludeFx.capabilities[0].nb.receipt, + blocks: receiptBlocks, + }) +} + +/** + * @param {API.Signer} id + * @param {API.Verifier} serviceDid + * @param {API.Receipt} receipt + */ +export function createConcludeInvocation(id, serviceDid, receipt) { + const receiptBlocks = [] + for (const block of receipt.iterateIPLDBlocks()) { + receiptBlocks.push(block) + } + const concludeAllocatefx = conclude.invoke({ + issuer: id, + audience: serviceDid, + with: id.toDIDKey(), + nb: { + receipt: receipt.link(), + }, + expiration: Infinity, + facts: [ + { + ...receiptBlocks.map((b) => b.cid), + }, + ], + }) + for (const block of receipt.iterateIPLDBlocks()) { + concludeAllocatefx.attach(block) + } + + return concludeAllocatefx +} diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index d89e06db6..683ae676f 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -2,9 +2,8 @@ import * as API from '../../src/types.js' import { equals } from 'uint8arrays' import pDefer from 'p-defer' import { Absentee } from '@ucanto/principal' -import { Message, Receipt } from '@ucanto/core' +import { Receipt } from '@ucanto/core' import { ed25519 } from '@ucanto/principal' -import { CAR } from '@ucanto/transport' import { sha256 } from 'multiformats/hashes/sha2' import * as BlobCapabilities from '@web3-storage/capabilities/blob' import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' @@ -16,7 +15,10 @@ import { provisionProvider } from '../helpers/utils.js' import { createServer, connect } from '../../src/lib.js' import { alice, bob, createSpace, registerSpace } from '../util.js' import { BlobSizeOutsideOfSupportedRangeName } from '../../src/blob/lib.js' -import { findBlock } from '../../src/ucan/conclude.js' +import { + createConcludeInvocation, + getConcludeReceipt, +} from '../../src/ucan/conclude.js' /** * @type {API.Tests} @@ -91,29 +93,14 @@ export const test = { const httpPutGetTask = await context.tasksStorage.get(putfx.cid) assert.ok(httpPutGetTask.ok) + const receipt = getConcludeReceipt(allocateUcanConcludefx) // validate that scheduled allocate task executed and has its receipt content - const getBlockRes = await findBlock( - // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.receipt, - allocateUcanConcludefx.iterateIPLDBlocks() - ) - if (getBlockRes.error) { - throw new Error('receipt block should exist in invocation') - } - const messageCar = CAR.codec.decode(getBlockRes.ok) - const message = Message.view({ - root: messageCar.roots[0].cid, - store: messageCar.blocks, - }) - - const receiptKey = Array.from(message.receipts.keys())[0] - const receipt = message.receipts.get(receiptKey) - assert.ok(receipt?.out) - assert.ok(receipt?.out.ok) + assert.ok(receipt.out) + assert.ok(receipt.out.ok) // @ts-expect-error receipt out is unknown - assert.equal(receipt?.out.ok?.size, size) + assert.equal(receipt.out.ok?.size, size) // @ts-expect-error receipt out is unknown - assert.ok(receipt?.out.ok?.address) + assert.ok(receipt.out.ok?.address) }, 'blob/add executes allocation and returns effects for allocate (and its receipt) and accept, but not for put when blob stored': async (assert, context) => { @@ -170,24 +157,7 @@ export const test = { if (!allocateUcanConcludefx || !putfx) { throw new Error('effects not provided') } - const getBlockRes = await findBlock( - // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.receipt, - allocateUcanConcludefx.iterateIPLDBlocks() - ) - if (getBlockRes.error) { - throw new Error('receipt block should exist in invocation') - } - const messageCar = CAR.codec.decode(getBlockRes.ok) - const message = Message.view({ - root: messageCar.roots[0].cid, - store: messageCar.blocks, - }) - const receiptKey = Array.from(message.receipts.keys())[0] - const receipt = message.receipts.get(receiptKey) - if (!receipt) { - throw new Error('receipt should be available') - } + const receipt = getConcludeReceipt(allocateUcanConcludefx) const receiptPutRes = await context.receiptsStorage.put(receipt) assert.ok(receiptPutRes.ok) @@ -221,22 +191,23 @@ export const test = { throw new Error('invocation failed', { cause: thirdBlobAdd }) } - // Validate receipt has now only 2 effects + // Validate receipt has now only 3 effects assert.ok(thirdBlobAdd.out.ok) assert.ok(thirdBlobAdd.fx.join) - assert.equal(thirdBlobAdd.fx.fork.length, 2) - - /** - * @type {import('@ucanto/interface').Invocation[]} - **/ - // @ts-expect-error read only effect - const thirdForkInvocations = thirdBlobAdd.fx.fork - // no put effect anymore - assert.ok( - !thirdForkInvocations.find( - (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can - ) - ) + // TODO + assert.equal(thirdBlobAdd.fx.fork.length, 3) + + // /** + // * @type {import('@ucanto/interface').Invocation[]} + // **/ + // // @ts-expect-error read only effect + // const thirdForkInvocations = thirdBlobAdd.fx.fork + // // no put effect anymore + // assert.ok( + // !thirdForkInvocations.find( + // (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can + // ) + // ) }, 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { @@ -826,30 +797,19 @@ export const test = { **/ // @ts-expect-error read only effect const forkInvocations = blobAdd.fx.fork + const allocatefx = forkInvocations.find( + (fork) => fork.capabilities[0].can === W3sBlobCapabilities.allocate.can + ) const allocateUcanConcludefx = forkInvocations.find( (fork) => fork.capabilities[0].can === UCAN.conclude.can ) const putfx = forkInvocations.find( (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can ) - if (!allocateUcanConcludefx || !putfx) { + if (!allocateUcanConcludefx || !putfx || !allocatefx) { throw new Error('effects not provided') } - const getBlockRes = await findBlock( - // @ts-expect-error object of type unknown - allocateUcanConcludefx.capabilities[0].nb.receipt, - allocateUcanConcludefx.iterateIPLDBlocks() - ) - if (getBlockRes.error) { - throw new Error('receipt block should exist in invocation') - } - const blobAllocateMessageCar = CAR.codec.decode(getBlockRes.ok) - const blobAllocateMessage = Message.view({ - root: blobAllocateMessageCar.roots[0].cid, - store: blobAllocateMessageCar.blocks, - }) - const receiptKey = Array.from(blobAllocateMessage.receipts.keys())[0] - const receipt = blobAllocateMessage.receipts.get(receiptKey) + const receipt = getConcludeReceipt(allocateUcanConcludefx) // Get `blob/allocate` receipt with address /** @@ -881,8 +841,12 @@ export const test = { digest, size, }, - url: address.url, - headers: address.headers, + url: { + 'ucan/await': ['.out.ok.address.url', allocatefx.cid], + }, + headers: { + 'ucan/await': ['.out.ok.address.headers', allocatefx.cid], + }, }, facts: putfx.facts, expiration: Infinity, @@ -896,38 +860,36 @@ export const test = { ok: {}, }, }) - const message = await Message.build({ receipts: [httpPutReceipt] }) - const messageCar = await CAR.outbound.encode(message) - const bytes = new Uint8Array(messageCar.body) - const messageLink = await CAR.codec.link(bytes) - - // Invoke `ucan/conclude` with `http/put` receipt - const httpPutConcludeInvocation = UCAN.conclude.invoke({ - issuer: alice, - audience: context.id, - with: alice.did(), - nb: { - receipt: messageLink, - }, - expiration: Infinity, - }) - httpPutConcludeInvocation.attach({ - bytes, - cid: messageLink, - }) + const httpPutConcludeInvocation = createConcludeInvocation( + alice, + context.id, + httpPutReceipt + ) const ucanConclude = await httpPutConcludeInvocation.execute(connection) if (!ucanConclude.out.ok) { throw new Error('invocation failed', { cause: blobAdd }) } // verify accept was scheduled + /** @type {import('@ucanto/interface').Invocation} */ const blobAcceptInvocation = await taskScheduled.promise - assert.ok(blobAcceptInvocation) - assert.equal(blobAdd.out.ok.site['ucan/await'][0], '.out.ok.site') + assert.equal(blobAcceptInvocation.capabilities.length, 1) + assert.equal( + blobAcceptInvocation.capabilities[0].can, + W3sBlobCapabilities.accept.can + ) + assert.ok(blobAcceptInvocation.capabilities[0].nb.exp) + assert.equal( + blobAcceptInvocation.capabilities[0].nb._put['ucan/await'][0], + '.out.ok' + ) assert.ok( - blobAdd.out.ok.site['ucan/await'][1].equals(blobAcceptInvocation.cid) + blobAcceptInvocation.capabilities[0].nb._put['ucan/await'][1].equals( + httpPutDelegation.cid + ) ) - assert.ok(blobAdd.fx.join?.link().equals(blobAcceptInvocation.cid)) + assert.ok(blobAcceptInvocation.capabilities[0].nb.blob) + // TODO: space check }, // TODO: Blob accept } From 6f2bf8f1ef2412b36627d18ff6ff92101b40b776 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 10 Apr 2024 17:15:39 +0200 Subject: [PATCH 18/27] fix: add expiration to allocate receipt and wire up space for accept --- packages/capabilities/src/types.ts | 1 + packages/upload-api/src/blob/accept.js | 6 +- packages/upload-api/src/blob/add.js | 22 +- packages/upload-api/src/blob/allocate.js | 34 +- packages/upload-api/src/ucan/conclude.js | 31 +- packages/upload-api/test/handlers/blob.js | 852 +++--------------- packages/upload-api/test/handlers/ucan.js | 165 +++- .../upload-api/test/handlers/web3.storage.js | 519 +++++++++++ .../test/handlers/web3.storage.spec.js | 4 + packages/upload-api/test/helpers/blob.js | 60 ++ 10 files changed, 959 insertions(+), 735 deletions(-) create mode 100644 packages/upload-api/test/handlers/web3.storage.js create mode 100644 packages/upload-api/test/handlers/web3.storage.spec.js create mode 100644 packages/upload-api/test/helpers/blob.js diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 52af012fb..5192cb96c 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -494,6 +494,7 @@ export interface BlobAllocateSuccess { export interface BlobAddress { url: ToString headers: Record + expiresAt: ISO8601Date } // If space has not enough space to allocate the blob. diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index 4c4afec28..f3ba18e6a 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -1,4 +1,5 @@ import * as Server from '@ucanto/server' +import * as DID from '@ipld/dag-ucan/did' import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import { Assert } from '@web3-storage/content-claims/capability' import { create as createLink } from 'multiformats/link' @@ -19,7 +20,7 @@ export function blobAcceptProvider(context) { return Server.provideAdvanced({ capability: W3sBlob.accept, handler: async ({ capability }) => { - const { blob } = capability.nb + const { blob, space } = capability.nb // If blob is not stored, we must fail const hasBlob = await context.blobsStorage.has(blob.digest) if (hasBlob.error) { @@ -36,8 +37,7 @@ export function blobAcceptProvider(context) { const locationClaim = await Assert.location .invoke({ issuer: context.id, - // TODO: we need space CID here - audience: context.id, + audience: DID.parse(space), with: context.id.toDIDKey(), nb: { content, diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 6275e6bba..fb918fa06 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -40,6 +40,7 @@ export function blobAddProvider(context) { // Schedule allocate const scheduleAllocateRes = await scheduleAllocate({ context, + blob, allocate: next.allocate, allocatefx: next.allocatefx, }) @@ -232,10 +233,12 @@ async function schedulePut({ context, putfx }) { * * @param {object} scheduleAllocateProps * @param {API.BlobServiceContext} scheduleAllocateProps.context + * @param {API.BlobModel} scheduleAllocateProps.blob * @param {API.IssuedInvocationView} scheduleAllocateProps.allocate * @param {API.Invocation} scheduleAllocateProps.allocatefx */ -async function scheduleAllocate({ context, allocate, allocatefx }) { +async function scheduleAllocate({ context, blob, allocate, allocatefx }) { + /** @type {import('@ucanto/interface').Receipt | undefined} */ let blobAllocateReceipt // Get receipt for `blob/allocate` if available, otherwise schedule invocation @@ -245,8 +248,21 @@ async function scheduleAllocate({ context, allocate, allocatefx }) { error: receiptGet.error, } } else if (receiptGet.ok) { - // TODO: check expired by adding to receipt? + // @ts-expect-error ts not able to cast receipt blobAllocateReceipt = receiptGet.ok + + // Verify if allocation is expired before "accepting" this receipt. + // Note that if there is no address, means it was already allocated successfully before + const expiresAt = blobAllocateReceipt?.out.ok?.address?.expiresAt + if (expiresAt && (new Date()).getTime() > (new Date(expiresAt)).getTime()) { + // if expired, we must see if blob was written to avoid allocating one more time + const hasBlobStore = await context.blobsStorage.has(blob.digest) + if (hasBlobStore.error) { + return hasBlobStore + } else if (!hasBlobStore.ok) { + blobAllocateReceipt = undefined + } + } } // if not already allocated (or expired) schedule `blob/allocate` @@ -338,7 +354,7 @@ async function createNextTasks({ context, blob, space, cause }) { const accept = W3sBlob.accept.invoke({ issuer: context.id, audience: context.id, - with: context.id.toDIDKey(), + with: context.id.did(), nb: { blob, exp: Number.MAX_SAFE_INTEGER, diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index 7ea2b985a..1ab39a37e 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -12,6 +12,7 @@ export function blobAllocateProvider(context) { W3sBlob.allocate, async ({ capability, invocation }) => { const { blob, cause, space } = capability.nb + let size = blob.size // Rate limiting validation // TODO: we should not produce rate limit error but rather suspend / queue task to be run after enforcing a limit without erroring @@ -55,17 +56,30 @@ export function blobAllocateProvider(context) { // added to the space and there is no allocation change. // If record exists but is expired, it can be re-written if (allocationInsert.error.name === 'RecordKeyConflict') { + size = 0 + } else { return { - ok: { size: 0 }, + error: allocationInsert.error, } } + } + + // Check if blob already exists + const hasBlobStore = await context.blobsStorage.has(blob.digest) + if (hasBlobStore.error) { + return hasBlobStore + } + + // If blob is stored, we can just allocate it to the space with the allocated size + if (hasBlobStore.ok) { return { - error: allocationInsert.error, + ok: { size }, } } // Get presigned URL for the write target const expiresIn = 60 * 60 * 24 // 1 day + const expiresAt = (new Date(Date.now() + expiresIn)).toISOString() const createUploadUrl = await context.blobsStorage.createUploadUrl( blob.digest, blob.size, @@ -75,27 +89,15 @@ export function blobAllocateProvider(context) { return createUploadUrl } - // Check if blob already exists - const hasBlobStore = await context.blobsStorage.has(blob.digest) - if (hasBlobStore.error) { - return hasBlobStore - } - - // If blob is stored, we can just allocate it to the space - if (hasBlobStore.ok) { - return { - ok: { size: blob.size }, - } - } - const address = { url: createUploadUrl.ok.url.toString(), headers: createUploadUrl.ok.headers, + expiresAt } return { ok: { - size: blob.size, + size, address, }, } diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index 66015dabe..50cff9d27 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -1,10 +1,11 @@ +import * as API from '../types.js' import { provide } from '@ucanto/server' import { Receipt } from '@ucanto/core' import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import * as HTTP from '@web3-storage/capabilities/http' import { conclude } from '@web3-storage/capabilities/ucan' +import { equals } from 'uint8arrays/equals' import { DecodeBlockOperationFailed } from '../errors.js' -import * as API from '../types.js' /** * @param {API.ConcludeServiceContext} context @@ -35,22 +36,40 @@ export const ucanConcludeProvider = ({ } // THIS IS A TEMPORARY HACK - // Schedule `blob/accept` if there is a `http/put` capability + // Schedule `blob/accept` if there is a `http/put` capabilities + // inside the invocation that this receipt comes from const scheduleRes = await Promise.all( httpPutTaskGetRes.ok.capabilities + // Go through invocation tasks and get all `http/put` .filter((cap) => cap.can === HTTP.put.can) - .map(async (cap) => { + // @ts-expect-error body exists in `http/put` but unknown type here + .map(async (/** @type {API.HTTPPut} */ cap) => { + // Get triggering task (blob/allocate) by checking blocking task from `url` + /** @type {API.UnknownLink} */ + // @ts-expect-error ts does not know how to get this + const blobAllocateTaskCid = cap.nb.url['ucan/await'][1] + const blobAllocateTaskGet = await tasksStorage.get(blobAllocateTaskCid) + if (blobAllocateTaskGet.error) { + return blobAllocateTaskGet + } + + /** @type {API.BlobAllocate} */ + // @ts-expect-error ts does not know how to get this + const allocateCapability = blobAllocateTaskGet.ok.capabilities.find( + // @ts-expect-error ts does not know how to get this + (/** @type {API.BlobAllocate} */ allocateCap) => + equals(allocateCap.nb.blob.digest, cap.nb.body.digest) && allocateCap.can === W3sBlob.allocate.can + ) + const blobAccept = await W3sBlob.accept .invoke({ issuer: id, audience: id, with: id.toDIDKey(), nb: { - // @ts-expect-error body exists in `http/put` but unknown type here blob: cap.nb.body, exp: Number.MAX_SAFE_INTEGER, - // TODO: corect space - space: id.toDIDKey(), + space: allocateCapability.nb.space, _put: { 'ucan/await': ['.out.ok', ranInvocation.link()], }, diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 683ae676f..741cdd2e1 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -1,30 +1,21 @@ import * as API from '../../src/types.js' -import { equals } from 'uint8arrays' -import pDefer from 'p-defer' -import { Absentee } from '@ucanto/principal' -import { Receipt } from '@ucanto/core' -import { ed25519 } from '@ucanto/principal' import { sha256 } from 'multiformats/hashes/sha2' +import { ed25519 } from '@ucanto/principal' +import { Receipt } from '@ucanto/core' import * as BlobCapabilities from '@web3-storage/capabilities/blob' -import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' import * as HTTPCapabilities from '@web3-storage/capabilities/http' -import * as UCAN from '@web3-storage/capabilities/ucan' -import { base64pad } from 'multiformats/bases/base64' -import { provisionProvider } from '../helpers/utils.js' import { createServer, connect } from '../../src/lib.js' -import { alice, bob, createSpace, registerSpace } from '../util.js' +import { alice, registerSpace } from '../util.js' import { BlobSizeOutsideOfSupportedRangeName } from '../../src/blob/lib.js' -import { - createConcludeInvocation, - getConcludeReceipt, -} from '../../src/ucan/conclude.js' +import { createConcludeInvocation } from '../../src/ucan/conclude.js' +import { parseBlobAddReceiptNext } from '../helpers/blob.js' /** * @type {API.Tests} */ export const test = { - 'blob/add executes allocation and returns effects for allocate (and its receipt), accept and put': + 'blob/add schedules allocation and returns effects for allocate (and its receipt), put and accept': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -58,51 +49,41 @@ export const test = { throw new Error('invocation failed', { cause: blobAdd }) } - // Validate receipt + // Validate receipt structure assert.ok(blobAdd.out.ok.site) assert.equal(blobAdd.out.ok.site['ucan/await'][0], '.out.ok.site') assert.ok( blobAdd.out.ok.site['ucan/await'][1].equals(blobAdd.fx.join?.link()) ) assert.ok(blobAdd.fx.join) - - /** - * @type {import('@ucanto/interface').Invocation[]} - **/ - // @ts-expect-error read only effect - const forkInvocations = blobAdd.fx.fork assert.equal(blobAdd.fx.fork.length, 3) - const allocatefx = forkInvocations.find( - (fork) => fork.capabilities[0].can === W3sBlobCapabilities.allocate.can - ) - const allocateUcanConcludefx = forkInvocations.find( - (fork) => fork.capabilities[0].can === UCAN.conclude.can - ) - const putfx = forkInvocations.find( - (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can - ) - if (!allocatefx || !allocateUcanConcludefx || !putfx) { - throw new Error('effects not provided') - } + + // validate receipt next + const next = parseBlobAddReceiptNext(blobAdd) + assert.ok(next.allocatefx) + assert.ok(next.putfx) + assert.ok(next.acceptfx) + assert.equal(next.concludefxs.length, 1) + assert.ok(next.allocateReceipt) + assert.ok(!next.putReceipt) + assert.ok(!next.acceptReceipt) // validate facts exist for `http/put` - assert.ok(putfx.facts.length) - assert.ok(putfx.facts[0]['keys']) + assert.ok(next.putfx.facts.length) + assert.ok(next.putfx.facts[0]['keys']) - // Validate `http/put` invocation stored - const httpPutGetTask = await context.tasksStorage.get(putfx.cid) + // Validate `http/put` invocation was stored + const httpPutGetTask = await context.tasksStorage.get(next.putfx.cid) assert.ok(httpPutGetTask.ok) - const receipt = getConcludeReceipt(allocateUcanConcludefx) // validate that scheduled allocate task executed and has its receipt content + const receipt = next.allocateReceipt assert.ok(receipt.out) assert.ok(receipt.out.ok) - // @ts-expect-error receipt out is unknown assert.equal(receipt.out.ok?.size, size) - // @ts-expect-error receipt out is unknown assert.ok(receipt.out.ok?.address) }, - 'blob/add executes allocation and returns effects for allocate (and its receipt) and accept, but not for put when blob stored': + 'blob/add schedules allocation only on first blob/add': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -137,28 +118,19 @@ export const test = { throw new Error('invocation failed', { cause: firstBlobAdd }) } - // Validate receipt - assert.ok(firstBlobAdd.out.ok) - assert.ok(firstBlobAdd.fx.join) - assert.equal(firstBlobAdd.fx.fork.length, 3) - - // Store allocate receipt - /** - * @type {import('@ucanto/interface').Invocation[]} - **/ - // @ts-expect-error read only effect - const forkInvocations = firstBlobAdd.fx.fork - const allocateUcanConcludefx = forkInvocations.find( - (fork) => fork.capabilities[0].can === UCAN.conclude.can - ) - const putfx = forkInvocations.find( - (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can - ) - if (!allocateUcanConcludefx || !putfx) { - throw new Error('effects not provided') - } - const receipt = getConcludeReceipt(allocateUcanConcludefx) - const receiptPutRes = await context.receiptsStorage.put(receipt) + // parse first receipt next + const firstNext = parseBlobAddReceiptNext(firstBlobAdd) + assert.ok(firstNext.allocatefx) + assert.ok(firstNext.putfx) + assert.ok(firstNext.acceptfx) + assert.equal(firstNext.concludefxs.length, 1) + assert.ok(firstNext.allocateReceipt) + assert.ok(!firstNext.putReceipt) + assert.ok(!firstNext.acceptReceipt) + + // Store allocate receipt to not re-schedule + // @ts-expect-error types unknown for next + const receiptPutRes = await context.receiptsStorage.put(firstNext.allocateReceipt) assert.ok(receiptPutRes.ok) // Invoke `blob/add` for the second time (without storing the blob) @@ -167,187 +139,22 @@ export const test = { throw new Error('invocation failed', { cause: secondBlobAdd }) } - // Validate receipt has still 3 effects - assert.ok(secondBlobAdd.out.ok) - assert.ok(secondBlobAdd.fx.join) - assert.equal(secondBlobAdd.fx.fork.length, 3) - - /** @type {import('@web3-storage/capabilities/types').BlobAddress} */ - // @ts-expect-error receipt type is unknown - const address = receipt.out.ok.address - - // Store the blob to the address - const goodPut = await fetch(address.url, { - method: 'PUT', - mode: 'cors', - body: data, - headers: address.headers, - }) - assert.equal(goodPut.status, 200, await goodPut.text()) - - // Invoke `blob/add` for the third time (after storing the blob) - const thirdBlobAdd = await invocation.execute(connection) - if (!thirdBlobAdd.out.ok) { - throw new Error('invocation failed', { cause: thirdBlobAdd }) - } - - // Validate receipt has now only 3 effects - assert.ok(thirdBlobAdd.out.ok) - assert.ok(thirdBlobAdd.fx.join) - // TODO - assert.equal(thirdBlobAdd.fx.fork.length, 3) - - // /** - // * @type {import('@ucanto/interface').Invocation[]} - // **/ - // // @ts-expect-error read only effect - // const thirdForkInvocations = thirdBlobAdd.fx.fork - // // no put effect anymore - // assert.ok( - // !thirdForkInvocations.find( - // (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can - // ) - // ) + // parse second receipt next + const secondNext = parseBlobAddReceiptNext(secondBlobAdd) + assert.ok(secondNext.allocatefx) + assert.ok(secondNext.putfx) + assert.ok(secondNext.acceptfx) + assert.equal(secondNext.concludefxs.length, 1) + assert.ok(secondNext.allocateReceipt) + assert.ok(!secondNext.putReceipt) + assert.ok(!secondNext.acceptReceipt) + // allocate receipt is from same invocation CID + assert.ok(firstNext.concludefxs[0].cid.equals(secondNext.concludefxs[0].cid)) }, - 'blob/add fails when a blob with size bigger than maximum size is added': + 'blob/add schedules allocation and returns effects for allocate, accept and put together with their receipts (when stored)': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // invoke `blob/add` - const invocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size: Number.MAX_SAFE_INTEGER, - }, - }, - proofs: [proof], - }) - const blobAdd = await invocation.execute(connection) - if (!blobAdd.out.error) { - throw new Error('invocation should have failed') - } - assert.ok(blobAdd.out.error, 'invocation should have failed') - assert.equal(blobAdd.out.error.name, BlobSizeOutsideOfSupportedRangeName) - }, - 'blob/allocate allocates to space and returns presigned url': async ( - assert, - context - ) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - - // Validate response - assert.equal(blobAllocate.out.ok.size, size) - assert.ok(blobAllocate.out.ok.address) - assert.ok(blobAllocate.out.ok.address?.headers) - assert.ok(blobAllocate.out.ok.address?.url) - assert.equal( - blobAllocate.out.ok.address?.headers?.['content-length'], - String(size) - ) - assert.deepEqual( - blobAllocate.out.ok.address?.headers?.['x-amz-checksum-sha256'], - base64pad.baseEncode(multihash.digest) - ) - - const url = - blobAllocate.out.ok.address?.url && - new URL(blobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const signedHeaders = url.searchParams.get('X-Amz-SignedHeaders') - - assert.equal( - signedHeaders, - 'content-length;host;x-amz-checksum-sha256', - 'content-length and checksum must be part of the signature' - ) - - // Validate allocation state - const spaceAllocations = await context.allocationsStorage.list(spaceDid) - assert.ok(spaceAllocations.ok) - assert.equal(spaceAllocations.ok?.size, 1) - const allocatedEntry = spaceAllocations.ok?.results[0] - if (!allocatedEntry) { - throw new Error('Expected presigned allocatedEntry in response') - } - assert.ok(equals(allocatedEntry.blob.digest, digest)) - assert.equal(allocatedEntry.blob.size, size) - - // Validate presigned url usage - const goodPut = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: data, - headers: blobAllocate.out.ok.address?.headers, - }) - - assert.equal(goodPut.status, 200, await goodPut.text()) - }, - 'blob/allocate does not allocate more space to already allocated content': - async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) @@ -361,21 +168,7 @@ export const test = { }) // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + const invocation = BlobCapabilities.add.invoke({ issuer: alice, audience: context.id, with: spaceDid, @@ -384,393 +177,139 @@ export const test = { digest, size, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, }, proofs: [proof], }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - - // second blob allocate invocation - const secondBlobAllocate = await serviceBlobAllocate.execute(connection) - if (!secondBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: secondBlobAllocate }) + // Invoke `blob/add` for the first time + const firstBlobAdd = await invocation.execute(connection) + if (!firstBlobAdd.out.ok) { + throw new Error('invocation failed', { cause: firstBlobAdd }) } - // Validate response - assert.equal(secondBlobAllocate.out.ok.size, 0) - assert.ok(!!blobAllocate.out.ok.address) - }, - 'blob/allocate can allocate to different space after write to one space': - async (assert, context) => { - const { proof: aliceProof, spaceDid: aliceSpaceDid } = - await registerSpace(alice, context) - const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( - bob, - context, - 'bob' - ) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) + // parse first receipt next + const firstNext = parseBlobAddReceiptNext(firstBlobAdd) + assert.ok(firstNext.allocatefx) + assert.ok(firstNext.putfx) + assert.ok(firstNext.acceptfx) + assert.equal(firstNext.concludefxs.length, 1) + assert.ok(firstNext.allocateReceipt) + assert.ok(!firstNext.putReceipt) + assert.ok(!firstNext.acceptReceipt) + + // Store allocate receipt to not re-schedule + // @ts-expect-error types unknown for next + const receiptPutRes = await context.receiptsStorage.put(firstNext.allocateReceipt) + assert.ok(receiptPutRes.ok) - // create `blob/add` invocations - const aliceBlobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: aliceSpaceDid, - nb: { - blob: { - digest, - size, - }, - }, - proofs: [aliceProof], - }) - const bobBlobAddInvocation = BlobCapabilities.add.invoke({ - issuer: bob, - audience: context.id, - with: bobSpaceDid, - nb: { - blob: { - digest, - size, - }, - }, - proofs: [bobProof], - }) + /** @type {import('@web3-storage/capabilities/types').BlobAddress} */ + // @ts-expect-error receipt type is unknown + const address = firstNext.allocateReceipt.out.ok.address - // invoke `service/blob/allocate` capabilities on alice space - const aliceServiceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: aliceSpaceDid, - nb: { - blob: { - digest, - size, - }, - cause: (await aliceBlobAddInvocation.delegate()).cid, - space: aliceSpaceDid, - }, - proofs: [aliceProof], - }) - const aliceBlobAllocate = await aliceServiceBlobAllocate.execute( - connection - ) - if (!aliceBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: aliceBlobAllocate }) - } - // there is address to write - assert.ok(aliceBlobAllocate.out.ok.address) - assert.equal(aliceBlobAllocate.out.ok.size, size) - - // write to presigned url - const url = - aliceBlobAllocate.out.ok.address?.url && - new URL(aliceBlobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const goodPut = await fetch(url, { + // Store the blob to the address + const goodPut = await fetch(address.url, { method: 'PUT', mode: 'cors', body: data, - headers: aliceBlobAllocate.out.ok.address?.headers, + headers: address.headers, }) - assert.equal(goodPut.status, 200, await goodPut.text()) - // invoke `service/blob/allocate` capabilities on bob space - const bobServiceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ - issuer: bob, - audience: context.id, - with: bobSpaceDid, - nb: { - blob: { - digest, - size, - }, - cause: (await bobBlobAddInvocation.delegate()).cid, - space: bobSpaceDid, - }, - proofs: [bobProof], - }) - const bobBlobAllocate = await bobServiceBlobAllocate.execute(connection) - if (!bobBlobAllocate.out.ok) { - throw new Error('invocation failed', { cause: bobBlobAllocate }) + // Invoke `blob/add` for the second time (after storing the blob but not invoking conclude) + const secondBlobAdd = await invocation.execute(connection) + if (!secondBlobAdd.out.ok) { + throw new Error('invocation failed', { cause: secondBlobAdd }) } - // there is no address to write - assert.ok(!bobBlobAllocate.out.ok.address) - assert.equal(bobBlobAllocate.out.ok.size, size) - - // Validate allocation state - const aliceSpaceAllocations = await context.allocationsStorage.list( - aliceSpaceDid - ) - assert.ok(aliceSpaceAllocations.ok) - assert.equal(aliceSpaceAllocations.ok?.size, 1) - - const bobSpaceAllocations = await context.allocationsStorage.list( - bobSpaceDid - ) - assert.ok(bobSpaceAllocations.ok) - assert.equal(bobSpaceAllocations.ok?.size, 1) - }, - 'blob/allocate creates presigned url that can only PUT a payload with right length': - async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, + // parse second receipt next + const secondNext = parseBlobAddReceiptNext(secondBlobAdd) + assert.ok(secondNext.allocatefx) + assert.ok(secondNext.putfx) + assert.ok(secondNext.acceptfx) + assert.equal(secondNext.concludefxs.length, 1) + assert.ok(secondNext.allocateReceipt) + assert.ok(!secondNext.putReceipt) + assert.ok(!secondNext.acceptReceipt) + + // Store blob/allocate given conclude needs it to schedule blob/accept + // Store allocate task to be fetchable from allocate + await context.tasksStorage.put(secondNext.allocatefx) + + // Invoke `conclude` with `http/put` receipt + const keys = secondNext.putfx.facts[0]['keys'] + // @ts-expect-error Argument of type 'unknown' is not assignable to parameter of type 'SignerArchive<`did:${string}:${string}`, SigAlg>' + const blobProvider = ed25519.from(keys) + const httpPut = HTTPCapabilities.put.invoke({ + issuer: blobProvider, + audience: blobProvider, + with: blobProvider.toDIDKey(), nb: { - blob: { + body: { digest, size, }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, + url: { + 'ucan/await': ['.out.ok.address.url', secondNext.allocatefx.cid], }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - // there is address to write - assert.ok(blobAllocate.out.ok.address) - assert.equal(blobAllocate.out.ok.size, size) - - // write to presigned url - const url = - blobAllocate.out.ok.address?.url && - new URL(blobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const contentLengthFailSignature = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: longer, - headers: { - ...blobAllocate.out.ok.address?.headers, - 'content-length': longer.byteLength.toString(10), - }, - }) - - assert.equal( - contentLengthFailSignature.status >= 400, - true, - 'should fail to upload as content-length differs from that used to sign the url' - ) - }, - 'blob/allocate creates presigned url that can only PUT a payload with exact bytes': - async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const other = new Uint8Array([10, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, + headers: { + 'ucan/await': ['.out.ok.address.headers', secondNext.allocatefx.cid], }, }, - proofs: [proof], + facts: secondNext.putfx.facts, + expiration: Infinity, }) - // invoke `service/blob/allocate` - const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, + const httpPutDelegation = await httpPut.delegate() + const httpPutReceipt = await Receipt.issue({ + issuer: blobProvider, + ran: httpPutDelegation.cid, + result: { + ok: {}, }, - proofs: [proof], }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) + const httpPutConcludeInvocation = createConcludeInvocation( + alice, + context.id, + httpPutReceipt + ) + const ucanConclude = await httpPutConcludeInvocation.execute(connection) + if (!ucanConclude.out.ok) { + console.log('ucan conclude', ucanConclude.out.error) + throw new Error('invocation failed', { cause: ucanConclude.out }) } - // there is address to write - assert.ok(blobAllocate.out.ok.address) - assert.equal(blobAllocate.out.ok.size, size) - - // write to presigned url - const url = - blobAllocate.out.ok.address?.url && - new URL(blobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') + + // Invoke `blob/add` for the third time (after invoking conclude) + const thirdBlobAdd = await invocation.execute(connection) + if (!thirdBlobAdd.out.ok) { + throw new Error('invocation failed', { cause: thirdBlobAdd }) } - const failChecksum = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: other, - headers: blobAllocate.out.ok.address?.headers, - }) - assert.equal( - failChecksum.status, - 400, - 'should fail to upload any other data.' - ) + // parse third receipt next + const thirdNext = parseBlobAddReceiptNext(thirdBlobAdd) + assert.ok(thirdNext.allocatefx) + assert.ok(thirdNext.putfx) + assert.ok(thirdNext.acceptfx) + assert.equal(thirdNext.concludefxs.length, 3) + assert.ok(thirdNext.allocateReceipt) + assert.ok(thirdNext.putReceipt) + assert.ok(thirdNext.acceptReceipt) + + assert.ok(thirdNext.allocateReceipt.out.ok?.address) + assert.deepEqual(thirdNext.putReceipt?.out.ok, {}) + assert.ok(thirdNext.acceptReceipt?.out.ok?.site) }, - 'blob/allocate disallowed if invocation fails access verification': async ( - assert, - context - ) => { - const { proof, space, spaceDid } = await createSpace(alice) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, - }, - proofs: [proof], - }) - - // invoke `service/blob/allocate` - const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - assert.ok(blobAllocate.out.error) - assert.equal(blobAllocate.out.error?.message.includes('no storage'), true) - - // Register space and retry - const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) - const providerAdd = await provisionProvider({ - service: /** @type {API.Signer>} */ (context.signer), - agent: alice, - space, - account, - connection, - }) - assert.ok(providerAdd.out.ok) - - const retryBlobAllocate = await serviceBlobAllocate.execute(connection) - assert.equal(retryBlobAllocate.out.error, undefined) - }, - 'blob/accept is executed once ucan/conclude is invoked with the blob put receipt and blob was sent': + 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { - const taskScheduled = pDefer() const { proof, spaceDid } = await registerSpace(alice, context) // prepare data const data = new Uint8Array([11, 22, 34, 44, 55]) const multihash = await sha256.digest(data) const digest = multihash.bytes - const size = data.byteLength // create service connection const connection = connect({ id: context.id, - channel: createServer({ - ...context, - tasksScheduler: { - schedule: (invocation) => { - taskScheduled.resolve(invocation) - - return Promise.resolve({ - ok: {}, - }) - }, - }, - }), + channel: createServer(context), }) // invoke `blob/add` @@ -781,115 +320,16 @@ export const test = { nb: { blob: { digest, - size, + size: Number.MAX_SAFE_INTEGER, }, }, proofs: [proof], }) const blobAdd = await invocation.execute(connection) - if (!blobAdd.out.ok) { - throw new Error('invocation failed', { cause: blobAdd }) - } - - // Get receipt relevant content - /** - * @type {import('@ucanto/interface').Invocation[]} - **/ - // @ts-expect-error read only effect - const forkInvocations = blobAdd.fx.fork - const allocatefx = forkInvocations.find( - (fork) => fork.capabilities[0].can === W3sBlobCapabilities.allocate.can - ) - const allocateUcanConcludefx = forkInvocations.find( - (fork) => fork.capabilities[0].can === UCAN.conclude.can - ) - const putfx = forkInvocations.find( - (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can - ) - if (!allocateUcanConcludefx || !putfx || !allocatefx) { - throw new Error('effects not provided') - } - const receipt = getConcludeReceipt(allocateUcanConcludefx) - - // Get `blob/allocate` receipt with address - /** - * @type {import('@web3-storage/capabilities/types').BlobAddress} - **/ - // @ts-expect-error receipt out is unknown - const address = receipt?.out.ok?.address - assert.ok(address) - - // Write blob - const goodPut = await fetch(address.url, { - method: 'PUT', - mode: 'cors', - body: data, - headers: address?.headers, - }) - assert.equal(goodPut.status, 200, await goodPut.text()) - - // Create `http/put` receipt - const keys = putfx.facts[0]['keys'] - // @ts-expect-error Argument of type 'unknown' is not assignable to parameter of type 'SignerArchive<`did:${string}:${string}`, SigAlg>' - const blobProvider = ed25519.from(keys) - const httpPut = HTTPCapabilities.put.invoke({ - issuer: blobProvider, - audience: blobProvider, - with: blobProvider.toDIDKey(), - nb: { - body: { - digest, - size, - }, - url: { - 'ucan/await': ['.out.ok.address.url', allocatefx.cid], - }, - headers: { - 'ucan/await': ['.out.ok.address.headers', allocatefx.cid], - }, - }, - facts: putfx.facts, - expiration: Infinity, - }) - - const httpPutDelegation = await httpPut.delegate() - const httpPutReceipt = await Receipt.issue({ - issuer: blobProvider, - ran: httpPutDelegation.cid, - result: { - ok: {}, - }, - }) - const httpPutConcludeInvocation = createConcludeInvocation( - alice, - context.id, - httpPutReceipt - ) - const ucanConclude = await httpPutConcludeInvocation.execute(connection) - if (!ucanConclude.out.ok) { - throw new Error('invocation failed', { cause: blobAdd }) + if (!blobAdd.out.error) { + throw new Error('invocation should have failed') } - - // verify accept was scheduled - /** @type {import('@ucanto/interface').Invocation} */ - const blobAcceptInvocation = await taskScheduled.promise - assert.equal(blobAcceptInvocation.capabilities.length, 1) - assert.equal( - blobAcceptInvocation.capabilities[0].can, - W3sBlobCapabilities.accept.can - ) - assert.ok(blobAcceptInvocation.capabilities[0].nb.exp) - assert.equal( - blobAcceptInvocation.capabilities[0].nb._put['ucan/await'][0], - '.out.ok' - ) - assert.ok( - blobAcceptInvocation.capabilities[0].nb._put['ucan/await'][1].equals( - httpPutDelegation.cid - ) - ) - assert.ok(blobAcceptInvocation.capabilities[0].nb.blob) - // TODO: space check + assert.ok(blobAdd.out.error, 'invocation should have failed') + assert.equal(blobAdd.out.error.name, BlobSizeOutsideOfSupportedRangeName) }, - // TODO: Blob accept } diff --git a/packages/upload-api/test/handlers/ucan.js b/packages/upload-api/test/handlers/ucan.js index 1686decd4..50e06c19b 100644 --- a/packages/upload-api/test/handlers/ucan.js +++ b/packages/upload-api/test/handlers/ucan.js @@ -1,6 +1,19 @@ import * as API from '../../src/types.js' -import { alice, bob, mallory } from '../util.js' import { UCAN, Console } from '@web3-storage/capabilities' +import pDefer from 'p-defer' +import { Receipt } from '@ucanto/core' +import { ed25519 } from '@ucanto/principal' +import { sha256 } from 'multiformats/hashes/sha2' +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' +import * as HTTPCapabilities from '@web3-storage/capabilities/http' + +import { createServer, connect } from '../../src/lib.js' +import { alice, bob, mallory, registerSpace } from '../util.js' +import { + createConcludeInvocation, + getConcludeReceipt, +} from '../../src/ucan/conclude.js' /** * @type {API.Tests} @@ -352,4 +365,154 @@ export const test = { assert.ok(String(revoke.out.error?.message).match(/Constrain violation/)) }, + 'ucan/conclude schedules web3.storage/blob/accept if invoked with the blob put receipt': + async (assert, context) => { + const taskScheduled = pDefer() + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer({ + ...context, + tasksScheduler: { + schedule: (invocation) => { + taskScheduled.resolve(invocation) + + return Promise.resolve({ + ok: {}, + }) + }, + }, + }), + }) + + // invoke `blob/add` + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const blobAdd = await blobAddInvocation.execute(connection) + if (!blobAdd.out.ok) { + throw new Error('invocation failed', { cause: blobAdd }) + } + + // Get receipt relevant content + /** + * @type {import('@ucanto/interface').Invocation[]} + **/ + // @ts-expect-error read only effect + const forkInvocations = blobAdd.fx.fork + const allocatefx = forkInvocations.find( + (fork) => fork.capabilities[0].can === W3sBlobCapabilities.allocate.can + ) + const allocateUcanConcludefx = forkInvocations.find( + (fork) => fork.capabilities[0].can === UCAN.conclude.can + ) + const putfx = forkInvocations.find( + (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can + ) + if (!allocateUcanConcludefx || !putfx || !allocatefx) { + throw new Error('effects not provided') + } + const receipt = getConcludeReceipt(allocateUcanConcludefx) + + // Get `web3.storage/blob/allocate` receipt with address + /** + * @type {import('@web3-storage/capabilities/types').BlobAddress} + **/ + // @ts-expect-error receipt out is unknown + const address = receipt?.out.ok?.address + assert.ok(address) + + + // Store allocate task to be fetchable from allocate + await context.tasksStorage.put(allocatefx) + + // Write blob + const goodPut = await fetch(address.url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: address?.headers, + }) + assert.equal(goodPut.status, 200, await goodPut.text()) + + // Create `http/put` receipt + const keys = putfx.facts[0]['keys'] + // @ts-expect-error Argument of type 'unknown' is not assignable to parameter of type 'SignerArchive<`did:${string}:${string}`, SigAlg>' + const blobProvider = ed25519.from(keys) + const httpPut = HTTPCapabilities.put.invoke({ + issuer: blobProvider, + audience: blobProvider, + with: blobProvider.toDIDKey(), + nb: { + body: { + digest, + size, + }, + url: { + 'ucan/await': ['.out.ok.address.url', allocatefx.cid], + }, + headers: { + 'ucan/await': ['.out.ok.address.headers', allocatefx.cid], + }, + }, + facts: putfx.facts, + expiration: Infinity, + }) + + const httpPutDelegation = await httpPut.delegate() + const httpPutReceipt = await Receipt.issue({ + issuer: blobProvider, + ran: httpPutDelegation.cid, + result: { + ok: {}, + }, + }) + const httpPutConcludeInvocation = createConcludeInvocation( + alice, + context.id, + httpPutReceipt + ) + const ucanConclude = await httpPutConcludeInvocation.execute(connection) + if (!ucanConclude.out.ok) { + throw new Error('invocation failed', { cause: blobAdd }) + } + + // verify accept was scheduled + /** @type {import('@ucanto/interface').Invocation} */ + const blobAcceptInvocation = await taskScheduled.promise + assert.equal(blobAcceptInvocation.capabilities.length, 1) + assert.equal( + blobAcceptInvocation.capabilities[0].can, + W3sBlobCapabilities.accept.can + ) + assert.ok(blobAcceptInvocation.capabilities[0].nb.exp) + assert.equal( + blobAcceptInvocation.capabilities[0].nb._put['ucan/await'][0], + '.out.ok' + ) + assert.ok( + blobAcceptInvocation.capabilities[0].nb._put['ucan/await'][1].equals( + httpPutDelegation.cid + ) + ) + assert.ok(blobAcceptInvocation.capabilities[0].nb.blob) + assert.equal(blobAcceptInvocation.capabilities[0].nb.space, spaceDid) + }, } diff --git a/packages/upload-api/test/handlers/web3.storage.js b/packages/upload-api/test/handlers/web3.storage.js new file mode 100644 index 000000000..15eb65ff4 --- /dev/null +++ b/packages/upload-api/test/handlers/web3.storage.js @@ -0,0 +1,519 @@ +import * as API from '../../src/types.js' +import { equals } from 'uint8arrays' +import { Absentee } from '@ucanto/principal' +import { sha256 } from 'multiformats/hashes/sha2' +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' +import { base64pad } from 'multiformats/bases/base64' + +import { provisionProvider } from '../helpers/utils.js' +import { createServer, connect } from '../../src/lib.js' +import { alice, bob, createSpace, registerSpace } from '../util.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'web3.storage/blob/allocate allocates to space and returns presigned url': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + + // invoke `web3.storage/blob/allocate` + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + + // Validate response + assert.equal(blobAllocate.out.ok.size, size) + assert.ok(blobAllocate.out.ok.address) + assert.ok(blobAllocate.out.ok.address?.headers) + assert.ok(blobAllocate.out.ok.address?.url) + assert.equal( + blobAllocate.out.ok.address?.headers?.['content-length'], + String(size) + ) + assert.deepEqual( + blobAllocate.out.ok.address?.headers?.['x-amz-checksum-sha256'], + base64pad.baseEncode(multihash.digest) + ) + + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const signedHeaders = url.searchParams.get('X-Amz-SignedHeaders') + + assert.equal( + signedHeaders, + 'content-length;host;x-amz-checksum-sha256', + 'content-length and checksum must be part of the signature' + ) + + // Validate allocation state + const spaceAllocations = await context.allocationsStorage.list(spaceDid) + assert.ok(spaceAllocations.ok) + assert.equal(spaceAllocations.ok?.size, 1) + const allocatedEntry = spaceAllocations.ok?.results[0] + if (!allocatedEntry) { + throw new Error('Expected presigned allocatedEntry in response') + } + assert.ok(equals(allocatedEntry.blob.digest, digest)) + assert.equal(allocatedEntry.blob.size, size) + + // Validate presigned url usage + const goodPut = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: blobAllocate.out.ok.address?.headers, + }) + + assert.equal(goodPut.status, 200, await goodPut.text()) + }, + 'web3.storage/blob/allocate does not allocate more space to already allocated content': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + + // invoke `web3.storage/blob/allocate` + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + + // second blob allocate invocation + const secondBlobAllocate = await serviceBlobAllocate.execute(connection) + if (!secondBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: secondBlobAllocate }) + } + + // Validate response + assert.equal(secondBlobAllocate.out.ok.size, 0) + assert.ok(!!blobAllocate.out.ok.address) + }, + 'web3.storage/blob/allocate can allocate to different space after write to one space': + async (assert, context) => { + const { proof: aliceProof, spaceDid: aliceSpaceDid } = + await registerSpace(alice, context) + const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( + bob, + context, + 'bob' + ) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocations + const aliceBlobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [aliceProof], + }) + const bobBlobAddInvocation = BlobCapabilities.add.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [bobProof], + }) + + // invoke `web3.storage/blob/allocate` capabilities on alice space + const aliceServiceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await aliceBlobAddInvocation.delegate()).cid, + space: aliceSpaceDid, + }, + proofs: [aliceProof], + }) + const aliceBlobAllocate = await aliceServiceBlobAllocate.execute( + connection + ) + if (!aliceBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: aliceBlobAllocate }) + } + // there is address to write + assert.ok(aliceBlobAllocate.out.ok.address) + assert.equal(aliceBlobAllocate.out.ok.size, size) + + // write to presigned url + const url = + aliceBlobAllocate.out.ok.address?.url && + new URL(aliceBlobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const goodPut = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: aliceBlobAllocate.out.ok.address?.headers, + }) + + assert.equal(goodPut.status, 200, await goodPut.text()) + + // invoke `web3.storage/blob/allocate` capabilities on bob space + const bobServiceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await bobBlobAddInvocation.delegate()).cid, + space: bobSpaceDid, + }, + proofs: [bobProof], + }) + const bobBlobAllocate = await bobServiceBlobAllocate.execute(connection) + if (!bobBlobAllocate.out.ok) { + throw new Error('invocation failed', { cause: bobBlobAllocate }) + } + // there is no address to write + assert.ok(!bobBlobAllocate.out.ok.address) + assert.equal(bobBlobAllocate.out.ok.size, size) + + // Validate allocation state + const aliceSpaceAllocations = await context.allocationsStorage.list( + aliceSpaceDid + ) + assert.ok(aliceSpaceAllocations.ok) + assert.equal(aliceSpaceAllocations.ok?.size, 1) + + const bobSpaceAllocations = await context.allocationsStorage.list( + bobSpaceDid + ) + assert.ok(bobSpaceAllocations.ok) + assert.equal(bobSpaceAllocations.ok?.size, 1) + }, + 'web3.storage/blob/allocate creates presigned url that can only PUT a payload with right length': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const longer = new Uint8Array([11, 22, 34, 44, 55, 66]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + + // invoke `web3.storage/blob/allocate` + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + // there is address to write + assert.ok(blobAllocate.out.ok.address) + assert.equal(blobAllocate.out.ok.size, size) + + // write to presigned url + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const contentLengthFailSignature = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: longer, + headers: { + ...blobAllocate.out.ok.address?.headers, + 'content-length': longer.byteLength.toString(10), + }, + }) + + assert.equal( + contentLengthFailSignature.status >= 400, + true, + 'should fail to upload as content-length differs from that used to sign the url' + ) + }, + 'web3.storage/blob/allocate creates presigned url that can only PUT a payload with exact bytes': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const other = new Uint8Array([10, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + + // invoke `web3.storage/blob/allocate` + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + // there is address to write + assert.ok(blobAllocate.out.ok.address) + assert.equal(blobAllocate.out.ok.size, size) + + // write to presigned url + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const failChecksum = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: other, + headers: blobAllocate.out.ok.address?.headers, + }) + + assert.equal( + failChecksum.status, + 400, + 'should fail to upload any other data.' + ) + }, + 'web3.storage/blob/allocate disallowed if invocation fails access verification': async ( + assert, + context + ) => { + const { proof, space, spaceDid } = await createSpace(alice) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + + // invoke `web3.storage/blob/allocate` + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, + }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + assert.ok(blobAllocate.out.error) + assert.equal(blobAllocate.out.error?.message.includes('no storage'), true) + + // Register space and retry + const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) + const providerAdd = await provisionProvider({ + service: /** @type {API.Signer>} */ (context.signer), + agent: alice, + space, + account, + connection, + }) + assert.ok(providerAdd.out.ok) + + const retryBlobAllocate = await serviceBlobAllocate.execute(connection) + assert.equal(retryBlobAllocate.out.error, undefined) + }, + // TODO: Blob accept +} diff --git a/packages/upload-api/test/handlers/web3.storage.spec.js b/packages/upload-api/test/handlers/web3.storage.spec.js new file mode 100644 index 000000000..b68050534 --- /dev/null +++ b/packages/upload-api/test/handlers/web3.storage.spec.js @@ -0,0 +1,4 @@ +import { test } from '../test.js' +import * as W3s from './web3.storage.js' + +test({ 'web3.storage/*': W3s.test }) diff --git a/packages/upload-api/test/helpers/blob.js b/packages/upload-api/test/helpers/blob.js new file mode 100644 index 000000000..6e793ed22 --- /dev/null +++ b/packages/upload-api/test/helpers/blob.js @@ -0,0 +1,60 @@ +import * as API from '../../src/types.js' + +import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' +import * as HTTPCapabilities from '@web3-storage/capabilities/http' +import * as UCAN from '@web3-storage/capabilities/ucan' + +import { + getConcludeReceipt, +} from '../../src/ucan/conclude.js' + +/** + * @param {API.Receipt} receipt + */ +export function parseBlobAddReceiptNext (receipt) { + // Get invocations next + /** + * @type {import('@ucanto/interface').Invocation[]} + **/ + // @ts-expect-error read only effect + const forkInvocations = receipt.fx.fork + const allocatefx = forkInvocations.find( + (fork) => fork.capabilities[0].can === W3sBlobCapabilities.allocate.can + ) + const concludefxs = forkInvocations.filter( + (fork) => fork.capabilities[0].can === UCAN.conclude.can + ) + const putfx = forkInvocations.find( + (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can + ) + const acceptfx = receipt.fx.join + if (!allocatefx || !concludefxs.length || !putfx || !acceptfx) { + throw new Error('mandatory effects not received') + } + + // Decode receipts available + const nextReceipts = concludefxs.map(fx => getConcludeReceipt(fx)) + /** @type {API.Receipt | undefined} */ + // @ts-expect-error types unknown for next + const allocateReceipt = nextReceipts.find(receipt => receipt.ran.link().equals(allocatefx.cid)) + /** @type {API.Receipt<{}, API.Failure> | undefined} */ + // @ts-expect-error types unknown for next + const putReceipt = nextReceipts.find(receipt => receipt.ran.link().equals(putfx.cid)) + /** @type {API.Receipt | undefined} */ + // @ts-expect-error types unknown for next + const acceptReceipt = nextReceipts.find(receipt => receipt.ran.link().equals(acceptfx.link())) + + if (!allocateReceipt) { + throw new Error('mandatory effects not received') + } + + return { + allocatefx, + allocateReceipt, + concludefxs, + putfx, + putReceipt, + acceptfx, + acceptReceipt + } +} From dc5e8217a71e7b0669e3b42d5ec169c8a23e8a5f Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 11 Apr 2024 09:14:13 +0200 Subject: [PATCH 19/27] chore: re-enable attw --- .github/workflows/w3up-client.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/w3up-client.yml b/.github/workflows/w3up-client.yml index a469b0f98..cd2a01196 100644 --- a/.github/workflows/w3up-client.yml +++ b/.github/workflows/w3up-client.yml @@ -39,7 +39,8 @@ jobs: node-version: ${{ matrix.node_version }} registry-url: https://registry.npmjs.org/ cache: 'pnpm' - - run: pnpm install + - run: pnpm --filter '@web3-storage/w3up-client...' install + - run: pnpm --filter '@web3-storage/w3up-client' attw - uses: ./packages/w3up-client/.github/actions/test with: w3up-client-dir: ./packages/w3up-client/ From 0cc774ac9c2b3e19613949a15581e21c8e550758 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 11 Apr 2024 12:37:56 +0200 Subject: [PATCH 20/27] chore: rename exp to ttl and make optional for blob accept --- .../capabilities/src/web3.storage/blob.js | 6 +- packages/upload-api/src/blob/add.js | 3 +- packages/upload-api/src/blob/allocate.js | 4 +- packages/upload-api/src/ucan/conclude.js | 33 +- packages/upload-api/test/handlers/blob.js | 151 ++++---- packages/upload-api/test/handlers/ucan.js | 2 - .../upload-api/test/handlers/web3.storage.js | 322 +++++++++--------- packages/upload-api/test/helpers/blob.js | 24 +- 8 files changed, 267 insertions(+), 278 deletions(-) diff --git a/packages/capabilities/src/web3.storage/blob.js b/packages/capabilities/src/web3.storage/blob.js index 601b37192..42085380f 100644 --- a/packages/capabilities/src/web3.storage/blob.js +++ b/packages/capabilities/src/web3.storage/blob.js @@ -80,9 +80,9 @@ export const accept = capability({ */ blob: content, /** - * Expiration of location site. + * Content location commitment time to live, which will be encoded as expiry of the issued location claim. */ - exp: Schema.integer(), + ttl: Schema.integer().optional(), /** * DID of the user space where allocation took place */ @@ -96,7 +96,7 @@ export const accept = capability({ return ( and(equalWith(claim, from)) || and(equalBlob(claim, from)) || - and(equal(claim.nb.exp, from.nb.exp, 'exp')) || + and(equal(claim.nb.ttl, from.nb.ttl, 'ttl')) || and(equal(claim.nb.space, from.nb.space, 'space')) || ok({}) ) diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index fb918fa06..92692647d 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -254,7 +254,7 @@ async function scheduleAllocate({ context, blob, allocate, allocatefx }) { // Verify if allocation is expired before "accepting" this receipt. // Note that if there is no address, means it was already allocated successfully before const expiresAt = blobAllocateReceipt?.out.ok?.address?.expiresAt - if (expiresAt && (new Date()).getTime() > (new Date(expiresAt)).getTime()) { + if (expiresAt && new Date().getTime() > new Date(expiresAt).getTime()) { // if expired, we must see if blob was written to avoid allocating one more time const hasBlobStore = await context.blobsStorage.has(blob.digest) if (hasBlobStore.error) { @@ -357,7 +357,6 @@ async function createNextTasks({ context, blob, space, cause }) { with: context.id.did(), nb: { blob, - exp: Number.MAX_SAFE_INTEGER, space, _put: { 'ucan/await': ['.out.ok', putfx.link()] }, }, diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index 1ab39a37e..fed8121b7 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -79,7 +79,7 @@ export function blobAllocateProvider(context) { // Get presigned URL for the write target const expiresIn = 60 * 60 * 24 // 1 day - const expiresAt = (new Date(Date.now() + expiresIn)).toISOString() + const expiresAt = new Date(Date.now() + expiresIn).toISOString() const createUploadUrl = await context.blobsStorage.createUploadUrl( blob.digest, blob.size, @@ -92,7 +92,7 @@ export function blobAllocateProvider(context) { const address = { url: createUploadUrl.ok.url.toString(), headers: createUploadUrl.ok.headers, - expiresAt + expiresAt, } return { diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index 50cff9d27..210ffa15e 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -5,7 +5,6 @@ import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import * as HTTP from '@web3-storage/capabilities/http' import { conclude } from '@web3-storage/capabilities/ucan' import { equals } from 'uint8arrays/equals' -import { DecodeBlockOperationFailed } from '../errors.js' /** * @param {API.ConcludeServiceContext} context @@ -48,7 +47,9 @@ export const ucanConcludeProvider = ({ /** @type {API.UnknownLink} */ // @ts-expect-error ts does not know how to get this const blobAllocateTaskCid = cap.nb.url['ucan/await'][1] - const blobAllocateTaskGet = await tasksStorage.get(blobAllocateTaskCid) + const blobAllocateTaskGet = await tasksStorage.get( + blobAllocateTaskCid + ) if (blobAllocateTaskGet.error) { return blobAllocateTaskGet } @@ -58,9 +59,10 @@ export const ucanConcludeProvider = ({ const allocateCapability = blobAllocateTaskGet.ok.capabilities.find( // @ts-expect-error ts does not know how to get this (/** @type {API.BlobAllocate} */ allocateCap) => - equals(allocateCap.nb.blob.digest, cap.nb.body.digest) && allocateCap.can === W3sBlob.allocate.can + equals(allocateCap.nb.blob.digest, cap.nb.body.digest) && + allocateCap.can === W3sBlob.allocate.can ) - + const blobAccept = await W3sBlob.accept .invoke({ issuer: id, @@ -68,7 +70,6 @@ export const ucanConcludeProvider = ({ with: id.toDIDKey(), nb: { blob: cap.nb.body, - exp: Number.MAX_SAFE_INTEGER, space: allocateCapability.nb.space, _put: { 'ucan/await': ['.out.ok', ranInvocation.link()], @@ -94,28 +95,6 @@ export const ucanConcludeProvider = ({ } }) -/** - * @param {import('multiformats').UnknownLink} cid - * @param {IterableIterator>} blocks - * @returns {Promise>} - */ -export const findBlock = async (cid, blocks) => { - let bytes - for (const b of blocks) { - if (b.cid.equals(cid)) { - bytes = b.bytes - } - } - if (!bytes) { - return { - error: new DecodeBlockOperationFailed(`missing block: ${cid}`), - } - } - return { - ok: bytes, - } -} - /** * @param {import('@ucanto/interface').Invocation} concludeFx */ diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 741cdd2e1..87d1227fb 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -83,74 +83,80 @@ export const test = { assert.equal(receipt.out.ok?.size, size) assert.ok(receipt.out.ok?.address) }, - 'blob/add schedules allocation only on first blob/add': - async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const invocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, + 'blob/add schedules allocation only on first blob/add': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const invocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, }, - proofs: [proof], - }) - // Invoke `blob/add` for the first time - const firstBlobAdd = await invocation.execute(connection) - if (!firstBlobAdd.out.ok) { - throw new Error('invocation failed', { cause: firstBlobAdd }) - } - - // parse first receipt next - const firstNext = parseBlobAddReceiptNext(firstBlobAdd) - assert.ok(firstNext.allocatefx) - assert.ok(firstNext.putfx) - assert.ok(firstNext.acceptfx) - assert.equal(firstNext.concludefxs.length, 1) - assert.ok(firstNext.allocateReceipt) - assert.ok(!firstNext.putReceipt) - assert.ok(!firstNext.acceptReceipt) - - // Store allocate receipt to not re-schedule + }, + proofs: [proof], + }) + // Invoke `blob/add` for the first time + const firstBlobAdd = await invocation.execute(connection) + if (!firstBlobAdd.out.ok) { + throw new Error('invocation failed', { cause: firstBlobAdd }) + } + + // parse first receipt next + const firstNext = parseBlobAddReceiptNext(firstBlobAdd) + assert.ok(firstNext.allocatefx) + assert.ok(firstNext.putfx) + assert.ok(firstNext.acceptfx) + assert.equal(firstNext.concludefxs.length, 1) + assert.ok(firstNext.allocateReceipt) + assert.ok(!firstNext.putReceipt) + assert.ok(!firstNext.acceptReceipt) + + // Store allocate receipt to not re-schedule + const receiptPutRes = await context.receiptsStorage.put( // @ts-expect-error types unknown for next - const receiptPutRes = await context.receiptsStorage.put(firstNext.allocateReceipt) - assert.ok(receiptPutRes.ok) - - // Invoke `blob/add` for the second time (without storing the blob) - const secondBlobAdd = await invocation.execute(connection) - if (!secondBlobAdd.out.ok) { - throw new Error('invocation failed', { cause: secondBlobAdd }) - } - - // parse second receipt next - const secondNext = parseBlobAddReceiptNext(secondBlobAdd) - assert.ok(secondNext.allocatefx) - assert.ok(secondNext.putfx) - assert.ok(secondNext.acceptfx) - assert.equal(secondNext.concludefxs.length, 1) - assert.ok(secondNext.allocateReceipt) - assert.ok(!secondNext.putReceipt) - assert.ok(!secondNext.acceptReceipt) - // allocate receipt is from same invocation CID - assert.ok(firstNext.concludefxs[0].cid.equals(secondNext.concludefxs[0].cid)) - }, + firstNext.allocateReceipt + ) + assert.ok(receiptPutRes.ok) + + // Invoke `blob/add` for the second time (without storing the blob) + const secondBlobAdd = await invocation.execute(connection) + if (!secondBlobAdd.out.ok) { + throw new Error('invocation failed', { cause: secondBlobAdd }) + } + + // parse second receipt next + const secondNext = parseBlobAddReceiptNext(secondBlobAdd) + assert.ok(secondNext.allocatefx) + assert.ok(secondNext.putfx) + assert.ok(secondNext.acceptfx) + assert.equal(secondNext.concludefxs.length, 1) + assert.ok(secondNext.allocateReceipt) + assert.ok(!secondNext.putReceipt) + assert.ok(!secondNext.acceptReceipt) + // allocate receipt is from same invocation CID + assert.ok( + firstNext.concludefxs[0].cid.equals(secondNext.concludefxs[0].cid) + ) + }, 'blob/add schedules allocation and returns effects for allocate, accept and put together with their receipts (when stored)': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -197,8 +203,10 @@ export const test = { assert.ok(!firstNext.acceptReceipt) // Store allocate receipt to not re-schedule - // @ts-expect-error types unknown for next - const receiptPutRes = await context.receiptsStorage.put(firstNext.allocateReceipt) + const receiptPutRes = await context.receiptsStorage.put( + // @ts-expect-error types unknown for next + firstNext.allocateReceipt + ) assert.ok(receiptPutRes.ok) /** @type {import('@web3-storage/capabilities/types').BlobAddress} */ @@ -233,7 +241,7 @@ export const test = { // Store blob/allocate given conclude needs it to schedule blob/accept // Store allocate task to be fetchable from allocate await context.tasksStorage.put(secondNext.allocatefx) - + // Invoke `conclude` with `http/put` receipt const keys = secondNext.putfx.facts[0]['keys'] // @ts-expect-error Argument of type 'unknown' is not assignable to parameter of type 'SignerArchive<`did:${string}:${string}`, SigAlg>' @@ -251,7 +259,10 @@ export const test = { 'ucan/await': ['.out.ok.address.url', secondNext.allocatefx.cid], }, headers: { - 'ucan/await': ['.out.ok.address.headers', secondNext.allocatefx.cid], + 'ucan/await': [ + '.out.ok.address.headers', + secondNext.allocatefx.cid, + ], }, }, facts: secondNext.putfx.facts, diff --git a/packages/upload-api/test/handlers/ucan.js b/packages/upload-api/test/handlers/ucan.js index 50e06c19b..7758519f4 100644 --- a/packages/upload-api/test/handlers/ucan.js +++ b/packages/upload-api/test/handlers/ucan.js @@ -439,7 +439,6 @@ export const test = { const address = receipt?.out.ok?.address assert.ok(address) - // Store allocate task to be fetchable from allocate await context.tasksStorage.put(allocatefx) @@ -502,7 +501,6 @@ export const test = { blobAcceptInvocation.capabilities[0].can, W3sBlobCapabilities.accept.can ) - assert.ok(blobAcceptInvocation.capabilities[0].nb.exp) assert.equal( blobAcceptInvocation.capabilities[0].nb._put['ucan/await'][0], '.out.ok' diff --git a/packages/upload-api/test/handlers/web3.storage.js b/packages/upload-api/test/handlers/web3.storage.js index 15eb65ff4..1a9b8945e 100644 --- a/packages/upload-api/test/handlers/web3.storage.js +++ b/packages/upload-api/test/handlers/web3.storage.js @@ -14,107 +14,105 @@ import { alice, bob, createSpace, registerSpace } from '../util.js' * @type {API.Tests} */ export const test = { - 'web3.storage/blob/allocate allocates to space and returns presigned url': async ( - assert, - context - ) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, + 'web3.storage/blob/allocate allocates to space and returns presigned url': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, }, - }, - proofs: [proof], - }) - - // invoke `web3.storage/blob/allocate` - const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, + proofs: [proof], + }) + + // invoke `web3.storage/blob/allocate` + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - if (!blobAllocate.out.ok) { - throw new Error('invocation failed', { cause: blobAllocate }) - } - - // Validate response - assert.equal(blobAllocate.out.ok.size, size) - assert.ok(blobAllocate.out.ok.address) - assert.ok(blobAllocate.out.ok.address?.headers) - assert.ok(blobAllocate.out.ok.address?.url) - assert.equal( - blobAllocate.out.ok.address?.headers?.['content-length'], - String(size) - ) - assert.deepEqual( - blobAllocate.out.ok.address?.headers?.['x-amz-checksum-sha256'], - base64pad.baseEncode(multihash.digest) - ) - - const url = - blobAllocate.out.ok.address?.url && - new URL(blobAllocate.out.ok.address?.url) - if (!url) { - throw new Error('Expected presigned url in response') - } - const signedHeaders = url.searchParams.get('X-Amz-SignedHeaders') - - assert.equal( - signedHeaders, - 'content-length;host;x-amz-checksum-sha256', - 'content-length and checksum must be part of the signature' - ) - - // Validate allocation state - const spaceAllocations = await context.allocationsStorage.list(spaceDid) - assert.ok(spaceAllocations.ok) - assert.equal(spaceAllocations.ok?.size, 1) - const allocatedEntry = spaceAllocations.ok?.results[0] - if (!allocatedEntry) { - throw new Error('Expected presigned allocatedEntry in response') - } - assert.ok(equals(allocatedEntry.blob.digest, digest)) - assert.equal(allocatedEntry.blob.size, size) - - // Validate presigned url usage - const goodPut = await fetch(url, { - method: 'PUT', - mode: 'cors', - body: data, - headers: blobAllocate.out.ok.address?.headers, - }) - - assert.equal(goodPut.status, 200, await goodPut.text()) - }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + if (!blobAllocate.out.ok) { + throw new Error('invocation failed', { cause: blobAllocate }) + } + + // Validate response + assert.equal(blobAllocate.out.ok.size, size) + assert.ok(blobAllocate.out.ok.address) + assert.ok(blobAllocate.out.ok.address?.headers) + assert.ok(blobAllocate.out.ok.address?.url) + assert.equal( + blobAllocate.out.ok.address?.headers?.['content-length'], + String(size) + ) + assert.deepEqual( + blobAllocate.out.ok.address?.headers?.['x-amz-checksum-sha256'], + base64pad.baseEncode(multihash.digest) + ) + + const url = + blobAllocate.out.ok.address?.url && + new URL(blobAllocate.out.ok.address?.url) + if (!url) { + throw new Error('Expected presigned url in response') + } + const signedHeaders = url.searchParams.get('X-Amz-SignedHeaders') + + assert.equal( + signedHeaders, + 'content-length;host;x-amz-checksum-sha256', + 'content-length and checksum must be part of the signature' + ) + + // Validate allocation state + const spaceAllocations = await context.allocationsStorage.list(spaceDid) + assert.ok(spaceAllocations.ok) + assert.equal(spaceAllocations.ok?.size, 1) + const allocatedEntry = spaceAllocations.ok?.results[0] + if (!allocatedEntry) { + throw new Error('Expected presigned allocatedEntry in response') + } + assert.ok(equals(allocatedEntry.blob.digest, digest)) + assert.equal(allocatedEntry.blob.size, size) + + // Validate presigned url usage + const goodPut = await fetch(url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: blobAllocate.out.ok.address?.headers, + }) + + assert.equal(goodPut.status, 200, await goodPut.text()) + }, 'web3.storage/blob/allocate does not allocate more space to already allocated content': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -450,70 +448,70 @@ export const test = { 'should fail to upload any other data.' ) }, - 'web3.storage/blob/allocate disallowed if invocation fails access verification': async ( - assert, - context - ) => { - const { proof, space, spaceDid } = await createSpace(alice) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, + 'web3.storage/blob/allocate disallowed if invocation fails access verification': + async (assert, context) => { + const { proof, space, spaceDid } = await createSpace(alice) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, }, - }, - proofs: [proof], - }) - - // invoke `web3.storage/blob/allocate` - const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, + proofs: [proof], + }) + + // invoke `web3.storage/blob/allocate` + const serviceBlobAllocate = W3sBlobCapabilities.allocate.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + cause: (await blobAddInvocation.delegate()).cid, + space: spaceDid, }, - cause: (await blobAddInvocation.delegate()).cid, - space: spaceDid, - }, - proofs: [proof], - }) - const blobAllocate = await serviceBlobAllocate.execute(connection) - assert.ok(blobAllocate.out.error) - assert.equal(blobAllocate.out.error?.message.includes('no storage'), true) - - // Register space and retry - const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) - const providerAdd = await provisionProvider({ - service: /** @type {API.Signer>} */ (context.signer), - agent: alice, - space, - account, - connection, - }) - assert.ok(providerAdd.out.ok) - - const retryBlobAllocate = await serviceBlobAllocate.execute(connection) - assert.equal(retryBlobAllocate.out.error, undefined) - }, + proofs: [proof], + }) + const blobAllocate = await serviceBlobAllocate.execute(connection) + assert.ok(blobAllocate.out.error) + assert.equal(blobAllocate.out.error?.message.includes('no storage'), true) + + // Register space and retry + const account = Absentee.from({ + id: 'did:mailto:test.web3.storage:alice', + }) + const providerAdd = await provisionProvider({ + service: /** @type {API.Signer>} */ (context.signer), + agent: alice, + space, + account, + connection, + }) + assert.ok(providerAdd.out.ok) + + const retryBlobAllocate = await serviceBlobAllocate.execute(connection) + assert.equal(retryBlobAllocate.out.error, undefined) + }, // TODO: Blob accept } diff --git a/packages/upload-api/test/helpers/blob.js b/packages/upload-api/test/helpers/blob.js index 6e793ed22..4cc710108 100644 --- a/packages/upload-api/test/helpers/blob.js +++ b/packages/upload-api/test/helpers/blob.js @@ -4,14 +4,12 @@ import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/bl import * as HTTPCapabilities from '@web3-storage/capabilities/http' import * as UCAN from '@web3-storage/capabilities/ucan' -import { - getConcludeReceipt, -} from '../../src/ucan/conclude.js' +import { getConcludeReceipt } from '../../src/ucan/conclude.js' /** - * @param {API.Receipt} receipt + * @param {API.Receipt} receipt */ -export function parseBlobAddReceiptNext (receipt) { +export function parseBlobAddReceiptNext(receipt) { // Get invocations next /** * @type {import('@ucanto/interface').Invocation[]} @@ -33,16 +31,22 @@ export function parseBlobAddReceiptNext (receipt) { } // Decode receipts available - const nextReceipts = concludefxs.map(fx => getConcludeReceipt(fx)) + const nextReceipts = concludefxs.map((fx) => getConcludeReceipt(fx)) /** @type {API.Receipt | undefined} */ // @ts-expect-error types unknown for next - const allocateReceipt = nextReceipts.find(receipt => receipt.ran.link().equals(allocatefx.cid)) + const allocateReceipt = nextReceipts.find((receipt) => + receipt.ran.link().equals(allocatefx.cid) + ) /** @type {API.Receipt<{}, API.Failure> | undefined} */ // @ts-expect-error types unknown for next - const putReceipt = nextReceipts.find(receipt => receipt.ran.link().equals(putfx.cid)) + const putReceipt = nextReceipts.find((receipt) => + receipt.ran.link().equals(putfx.cid) + ) /** @type {API.Receipt | undefined} */ // @ts-expect-error types unknown for next - const acceptReceipt = nextReceipts.find(receipt => receipt.ran.link().equals(acceptfx.link())) + const acceptReceipt = nextReceipts.find((receipt) => + receipt.ran.link().equals(acceptfx.link()) + ) if (!allocateReceipt) { throw new Error('mandatory effects not received') @@ -55,6 +59,6 @@ export function parseBlobAddReceiptNext (receipt) { putfx, putReceipt, acceptfx, - acceptReceipt + acceptReceipt, } } From a40e9f255dd975b52796a6cc40d22d34ec90c1cd Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 11 Apr 2024 13:17:37 +0200 Subject: [PATCH 21/27] fix: address review comments for capabilities and types --- packages/capabilities/src/blob.js | 11 +++--- packages/capabilities/src/http.js | 5 ++- packages/capabilities/src/types.ts | 44 ++++++++++++------------ packages/upload-api/src/blob/accept.js | 4 +-- packages/upload-api/src/blob/lib.js | 7 ++-- packages/upload-api/src/ucan/conclude.js | 7 ++++ packages/upload-api/src/ucan/lib.js | 31 +++++++++++++++++ 7 files changed, 73 insertions(+), 36 deletions(-) create mode 100644 packages/upload-api/src/ucan/lib.js diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 8b42eed58..7d8b89506 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -42,7 +42,7 @@ export const content = Schema.struct({ */ digest: Schema.bytes(), /** - * Size of the Blob file to be stored. Service will provision write target + * Number of bytes contained by this blob. Service will provision write target * for this exact size. Attempt to write a larger Blob file will fail. */ size: Schema.integer(), @@ -50,10 +50,9 @@ export const content = Schema.struct({ /** * `blob/add` capability allows agent to store a Blob into a (memory) space - * identified by did:key in the `with` field. Agent must precompute Blob locally - * and provide it's multihash and size using `nb.blob` field, allowing - * a service to provision a write location for the agent to PUT or POST desired - * Blob into. + * identified by did:key in the `with` field. Agent should compute blob multihash + * and size and provide it under `nb.blob` field, allowing a service to provision + * a write location for the agent to PUT desired Blob into. */ export const add = capability({ can: 'blob/add', @@ -64,7 +63,7 @@ export const add = capability({ with: SpaceDID, nb: Schema.struct({ /** - * Blob to allocate on the space. + * Blob to be added on the space. */ blob: content, }), diff --git a/packages/capabilities/src/http.js b/packages/capabilities/src/http.js index a240f6aa0..a20b025f8 100644 --- a/packages/capabilities/src/http.js +++ b/packages/capabilities/src/http.js @@ -13,9 +13,8 @@ import { content } from './blob.js' import { equal, equalBody, equalWith, SpaceDID, Await, and } from './utils.js' /** - * `http/put` capability invocation MAY be performed by any agent on behalf of the subject. - * The `blob/add` provider MUST add `/http/put` effect and capture private key of the - * `subject` in the `meta` field so that any agent could perform it. + * `http/put` capability invocation MAY be performed by any authorized agent on behalf of the subject + * as long as they have referenced `body` content to do so. */ export const put = capability({ can: 'http/put', diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 5192cb96c..279105585 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -44,6 +44,10 @@ export type ISO8601Date = string export type { Unit, PieceLink } +export interface UCANAwait { + 'ucan/await': [Selector, Link] +} + /** * An IPLD Link that has the CAR codec code. */ @@ -460,9 +464,7 @@ export interface BlobModel { // Blob add export interface BlobAddSuccess { - site: { - 'ucan/await': ['.out.ok.site', Link] - } + site: UCANAwait<'.out.ok.site'> } export interface BlobSizeOutsideOfSupportedRange extends Ucanto.Failure { @@ -497,44 +499,42 @@ export interface BlobAddress { expiresAt: ISO8601Date } -// If space has not enough space to allocate the blob. -export interface BlobNotAllocableToSpace extends Ucanto.Failure { - name: 'BlobNotAllocableToSpace' +// If user space has not enough space to allocate the blob. +export interface NotEnoughStorageCapacity extends Ucanto.Failure { + name: 'NotEnoughStorageCapacity' } -export type BlobAllocateFailure = BlobNotAllocableToSpace | Ucanto.Failure +export type BlobAllocateFailure = NotEnoughStorageCapacity | Ucanto.Failure // Blob accept export interface BlobAcceptSuccess { + // A Link for a delegation with site commiment for the added blob. site: Link } -export interface BlobNotFound extends Ucanto.Failure { - name: 'BlobNotFound' +export interface AllocatedMemoryHadNotBeenWrittenTo extends Ucanto.Failure { + name: 'AllocatedMemoryHadNotBeenWrittenTo' } // TODO: We should type the store errors and add them here, instead of Ucanto.Failure -export type BlobAcceptFailure = BlobNotFound | Ucanto.Failure +export type BlobAcceptFailure = + | AllocatedMemoryHadNotBeenWrittenTo + | Ucanto.Failure // Storage errors -export type StoragePutError = StorageOperationError | EncodeRecordFailed -export type StorageGetError = - | StorageOperationError - | EncodeRecordFailed - | RecordNotFound +export type StoragePutError = StorageOperationError +export type StorageGetError = StorageOperationError | RecordNotFound +// Operation on a storage failed with unexpected error export interface StorageOperationError extends Error { name: 'StorageOperationFailed' } +// Record requested not found in the storage export interface RecordNotFound extends Error { name: 'RecordNotFound' } -export interface EncodeRecordFailed extends Error { - name: 'EncodeRecordFailed' -} - // Store export type Store = InferInvokedCapability export type StoreAdd = InferInvokedCapability @@ -680,11 +680,11 @@ export type UCANRevokeFailure = /** * Error is raised when receipt is received for unknown invocation */ -export interface ReceiptInvocationNotFound extends Ucanto.Failure { - name: 'ReceiptInvocationNotFound' +export interface ReferencedInvocationNotFound extends Ucanto.Failure { + name: 'ReferencedInvocationNotFound' } -export type UCANConcludeFailure = ReceiptInvocationNotFound | Ucanto.Failure +export type UCANConcludeFailure = ReferencedInvocationNotFound | Ucanto.Failure // Admin export type Admin = InferInvokedCapability diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index f3ba18e6a..f701f298d 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -7,7 +7,7 @@ import { Digest } from 'multiformats/hashes/digest' import { sha256 } from 'multiformats/hashes/sha2' import { CAR } from '@ucanto/core' import * as API from '../types.js' -import { BlobItemNotFound } from './lib.js' +import { AllocatedMemoryHadNotBeenWrittenTo } from './lib.js' const R2_REGION = 'auto' const R2_BUCKET = 'carpark-prod-0' @@ -25,7 +25,7 @@ export function blobAcceptProvider(context) { const hasBlob = await context.blobsStorage.has(blob.digest) if (hasBlob.error) { return { - error: new BlobItemNotFound(), + error: new AllocatedMemoryHadNotBeenWrittenTo(), } } diff --git a/packages/upload-api/src/blob/lib.js b/packages/upload-api/src/blob/lib.js index bbdaa621d..2b49c7e36 100644 --- a/packages/upload-api/src/blob/lib.js +++ b/packages/upload-api/src/blob/lib.js @@ -1,7 +1,8 @@ import { Failure } from '@ucanto/server' -export const BlobItemNotFoundName = 'BlobItemNotFound' -export class BlobItemNotFound extends Failure { +export const AllocatedMemoryHadNotBeenWrittenToName = + 'AllocatedMemoryHadNotBeenWrittenTo' +export class AllocatedMemoryHadNotBeenWrittenTo extends Failure { /** * @param {import('@ucanto/interface').DID} [space] */ @@ -11,7 +12,7 @@ export class BlobItemNotFound extends Failure { } get name() { - return BlobItemNotFoundName + return AllocatedMemoryHadNotBeenWrittenToName } describe() { diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index 210ffa15e..aaa3db6e8 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -6,6 +6,8 @@ import * as HTTP from '@web3-storage/capabilities/http' import { conclude } from '@web3-storage/capabilities/ucan' import { equals } from 'uint8arrays/equals' +import { ReferencedInvocationNotFound } from './lib.js' + /** * @param {API.ConcludeServiceContext} context * @returns {API.ServiceMethod} @@ -23,6 +25,11 @@ export const ucanConcludeProvider = ({ const ranInvocation = receipt.ran const httpPutTaskGetRes = await tasksStorage.get(ranInvocation.link()) if (httpPutTaskGetRes.error) { + if (httpPutTaskGetRes.error.name === 'RecordNotFound') { + return { + error: new ReferencedInvocationNotFound(ranInvocation.link()) + } + } return httpPutTaskGetRes } diff --git a/packages/upload-api/src/ucan/lib.js b/packages/upload-api/src/ucan/lib.js new file mode 100644 index 000000000..11b05fc8e --- /dev/null +++ b/packages/upload-api/src/ucan/lib.js @@ -0,0 +1,31 @@ +import { Failure } from '@ucanto/server' + +export const ReferencedInvocationNotFoundName = + 'ReferencedInvocationNotFound' +export class ReferencedInvocationNotFound extends Failure { + /** + * @param {import('@ucanto/interface').Link} [invocation] + */ + constructor(invocation) { + super() + this.invocation = invocation + } + + get name() { + return ReferencedInvocationNotFoundName + } + + describe() { + if (this.invocation) { + return `Invocation not found in ${this.invocation.toString()}` + } + return `Invocation not found` + } + + toJSON() { + return { + ...super.toJSON(), + invocation: this.invocation, + } + } +} From e5e60d56eda6c6138a0211d5303749dea8e61688 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 11 Apr 2024 13:43:17 +0200 Subject: [PATCH 22/27] fix: address review comments for upload api --- packages/upload-api/src/blob/accept.js | 7 +++---- packages/upload-api/src/blob/add.js | 6 ++---- packages/upload-api/src/blob/allocate.js | 24 +++++------------------- packages/upload-api/src/blob/lib.js | 24 +++++------------------- packages/upload-api/src/types/blob.ts | 4 ++-- packages/upload-api/src/ucan/conclude.js | 8 +++++--- 6 files changed, 22 insertions(+), 51 deletions(-) diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index f701f298d..9dc32bbff 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -5,7 +5,7 @@ import { Assert } from '@web3-storage/content-claims/capability' import { create as createLink } from 'multiformats/link' import { Digest } from 'multiformats/hashes/digest' import { sha256 } from 'multiformats/hashes/sha2' -import { CAR } from '@ucanto/core' +import { code as rawCode } from 'multiformats/codecs/raw' import * as API from '../types.js' import { AllocatedMemoryHadNotBeenWrittenTo } from './lib.js' @@ -31,11 +31,11 @@ export function blobAcceptProvider(context) { // TODO: we need to support multihash in claims, or specify hardcoded codec const digest = new Digest(sha256.code, 32, blob.digest, blob.digest) - const content = createLink(CAR.code, digest) + const content = createLink(rawCode, digest) const w3link = `https://w3s.link/ipfs/${content.toString()}?origin=r2://${R2_REGION}/${R2_BUCKET}` const locationClaim = await Assert.location - .invoke({ + .delegate({ issuer: context.id, audience: DID.parse(space), with: context.id.toDIDKey(), @@ -48,7 +48,6 @@ export function blobAcceptProvider(context) { }, expiration: Infinity, }) - .delegate() // Create result object /** @type {API.OkBuilder} */ diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 92692647d..117c66bcb 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -18,14 +18,12 @@ export function blobAddProvider(context) { handler: async ({ capability, invocation }) => { // Prepare context const { blob } = capability.nb - const space = /** @type {import('@ucanto/interface').DIDKey} */ ( - Server.DID.parse(capability.with).did() - ) + const space = capability.with // Verify blob is within accept size if (blob.size > context.maxUploadSize) { return { - error: new BlobSizeOutsideOfSupportedRange(context.maxUploadSize), + error: new BlobSizeOutsideOfSupportedRange(blob.size, context.maxUploadSize), } } diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index fed8121b7..7abd151a7 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -1,7 +1,6 @@ import * as Server from '@ucanto/server' import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import * as API from '../types.js' -import { ensureRateLimitAbove } from '../utils/rate-limits.js' /** * @param {API.W3ServiceContext} context @@ -14,23 +13,9 @@ export function blobAllocateProvider(context) { const { blob, cause, space } = capability.nb let size = blob.size - // Rate limiting validation - // TODO: we should not produce rate limit error but rather suspend / queue task to be run after enforcing a limit without erroring - const rateLimitResult = await ensureRateLimitAbove( - context.rateLimitsStorage, - [space], - 0 - ) - if (rateLimitResult.error) { - return { - error: { - name: 'RateLimited', - message: `${space} is blocked`, - }, - } - } - - // Has Storage provider validation + // We check if space has storage provider associated. If it does not + // we return `InsufficientStorage` error as storage capacity is considered + // to be 0. const result = await context.provisionsStorage.hasStorageProvider(space) if (result.error) { return result @@ -45,7 +30,8 @@ export function blobAllocateProvider(context) { } } - // Allocate in space, ignoring if already allocated + // Allocate memory space for the blob. If memory for this blob is + // already allocated, this allocates 0 bytes. const allocationInsert = await context.allocationsStorage.insert({ space, blob, diff --git a/packages/upload-api/src/blob/lib.js b/packages/upload-api/src/blob/lib.js index 2b49c7e36..556c74176 100644 --- a/packages/upload-api/src/blob/lib.js +++ b/packages/upload-api/src/blob/lib.js @@ -3,41 +3,26 @@ import { Failure } from '@ucanto/server' export const AllocatedMemoryHadNotBeenWrittenToName = 'AllocatedMemoryHadNotBeenWrittenTo' export class AllocatedMemoryHadNotBeenWrittenTo extends Failure { - /** - * @param {import('@ucanto/interface').DID} [space] - */ - constructor(space) { - super() - this.space = space - } get name() { return AllocatedMemoryHadNotBeenWrittenToName } describe() { - if (this.space) { - return `Blob not found in ${this.space}` - } return `Blob not found` } - - toJSON() { - return { - ...super.toJSON(), - space: this.space, - } - } } export const BlobSizeOutsideOfSupportedRangeName = 'BlobSizeOutsideOfSupportedRange' export class BlobSizeOutsideOfSupportedRange extends Failure { /** + * @param {Number} blobSize * @param {Number} maxUploadSize */ - constructor(maxUploadSize) { + constructor(blobSize, maxUploadSize) { super() + this.blobSize = blobSize this.maxUploadSize = maxUploadSize } @@ -46,13 +31,14 @@ export class BlobSizeOutsideOfSupportedRange extends Failure { } describe() { - return `Blob exceeded maximum size limit: ${this.maxUploadSize}, consider splitting it into blobs that fit limit.` + return `Blob size ${this.blobSize} exceeded maximum size limit: ${this.maxUploadSize}, consider splitting it into blobs that fit limit.` } toJSON() { return { ...super.toJSON(), maxUploadSize: this.maxUploadSize, + blobSize: this.blobSize, } } } diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index fcca50aef..5e5e8b806 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -31,7 +31,7 @@ export interface AllocationsStorage { ) => Promise, Failure>> } -export interface Blob { +export interface BlobModel { digest: BlobMultihash size: number } @@ -39,7 +39,7 @@ export interface Blob { export interface BlobAddInput { space: DID invocation: UnknownLink - blob: Blob + blob: BlobModel } export interface BlobAddOutput diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index aaa3db6e8..e6dd54046 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -53,7 +53,7 @@ export const ucanConcludeProvider = ({ // Get triggering task (blob/allocate) by checking blocking task from `url` /** @type {API.UnknownLink} */ // @ts-expect-error ts does not know how to get this - const blobAllocateTaskCid = cap.nb.url['ucan/await'][1] + const [,blobAllocateTaskCid] = cap.nb.url['ucan/await'] const blobAllocateTaskGet = await tasksStorage.get( blobAllocateTaskCid ) @@ -124,8 +124,10 @@ export function getConcludeReceipt(concludeFx) { */ export function createConcludeInvocation(id, serviceDid, receipt) { const receiptBlocks = [] + const receiptCids = [] for (const block of receipt.iterateIPLDBlocks()) { receiptBlocks.push(block) + receiptCids.push(block.cid) } const concludeAllocatefx = conclude.invoke({ issuer: id, @@ -137,11 +139,11 @@ export function createConcludeInvocation(id, serviceDid, receipt) { expiration: Infinity, facts: [ { - ...receiptBlocks.map((b) => b.cid), + ...receiptCids, }, ], }) - for (const block of receipt.iterateIPLDBlocks()) { + for (const block of receiptBlocks) { concludeAllocatefx.attach(block) } From f32d7abce1c1a88e247f5be2e6ae62d60762efc7 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 11 Apr 2024 14:35:19 +0200 Subject: [PATCH 23/27] refactor: make schedule functions the same as invocation creation --- packages/upload-api/src/blob/add.js | 396 ++++++++++------------ packages/upload-api/test/handlers/blob.js | 113 +++--- packages/upload-api/test/helpers/blob.js | 33 +- 3 files changed, 260 insertions(+), 282 deletions(-) diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 117c66bcb..117271f33 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -27,220 +27,112 @@ export function blobAddProvider(context) { } } - // Create next tasks - const next = await createNextTasks({ + // Create allocate task, get its receipt if available and execute it if necessary + const allocateRes = await allocate({ context, blob, space, cause: invocation.link(), }) + if (allocateRes.error) { + return allocateRes + } - // Schedule allocate - const scheduleAllocateRes = await scheduleAllocate({ + // Create put task and get its receipt if available + const putRes = await put({ context, blob, - allocate: next.allocate, - allocatefx: next.allocatefx, + allocateTask: allocateRes.ok.task }) - if (scheduleAllocateRes.error) { - return scheduleAllocateRes + if (putRes.error) { + return putRes } - // Schedule put - const schedulePutRes = await schedulePut({ + // Create accept task, get its receipt if available and execute if necessary and ready + const acceptRes = await accept({ context, - putfx: next.putfx, + blob, + space, + putTask: putRes.ok.task, + putReceipt: putRes.ok.receipt }) - if (schedulePutRes.error) { - return schedulePutRes + if (acceptRes.error) { + return acceptRes } // Create result object /** @type {API.OkBuilder} */ const result = Server.ok({ site: { - 'ucan/await': ['.out.ok.site', next.acceptfx.link()], + 'ucan/await': ['.out.ok.site', acceptRes.ok.task.link()], }, }) - // In case there is no receipt for concludePutfx, we can return - if (!schedulePutRes.ok.concludePutfx) { + // If there is no receipt for `http/put` we also still are pending receipt for `accept` + if (!putRes.ok.receipt || !acceptRes.ok.receipt) { return ( result // 1. System attempts to allocate memory in user space for the blob. - .fork(next.allocatefx) - .fork(scheduleAllocateRes.ok.concludeAllocatefx) + .fork(allocateRes.ok.task) + .fork(allocateRes.ok.receipt) // 2. System requests user agent (or anyone really) to upload the content // corresponding to the blob // via HTTP PUT to given location. - .fork(next.putfx) + .fork(putRes.ok.task) // 3. System will attempt to accept uploaded content that matches blob // multihash and size. - .join(next.acceptfx) + .join(acceptRes.ok.task) ) } - // schedule accept if there is http/put receipt available - const scheduleAcceptRes = await scheduleAccept({ - context, - accept: next.accept, - acceptfx: next.acceptfx, - }) - if (scheduleAcceptRes.error) { - return scheduleAcceptRes - } - - return scheduleAcceptRes.ok.concludeAcceptfx - ? result - // 1. System attempts to allocate memory in user space for the blob. - .fork(next.allocatefx) - .fork(scheduleAllocateRes.ok.concludeAllocatefx) - // 2. System requests user agent (or anyone really) to upload the content - // corresponding to the blob - // via HTTP PUT to given location. - .fork(next.putfx) - .fork(schedulePutRes.ok.concludePutfx) - // 3. System will attempt to accept uploaded content that matches blob - // multihash and size. - .join(next.acceptfx) - .fork(scheduleAcceptRes.ok.concludeAcceptfx) - : result - // 1. System attempts to allocate memory in user space for the blob. - .fork(next.allocatefx) - .fork(scheduleAllocateRes.ok.concludeAllocatefx) - // 2. System requests user agent (or anyone really) to upload the content - // corresponding to the blob - // via HTTP PUT to given location. - .fork(next.putfx) - .fork(schedulePutRes.ok.concludePutfx) - // 3. System will attempt to accept uploaded content that matches blob - // multihash and size. - .join(next.acceptfx) - }, + return result + // 1. System attempts to allocate memory in user space for the blob. + .fork(allocateRes.ok.task) + .fork(allocateRes.ok.receipt) + // 2. System requests user agent (or anyone really) to upload the content + // corresponding to the blob + // via HTTP PUT to given location. + .fork(putRes.ok.task) + .fork(putRes.ok.receipt) + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. + .join(acceptRes.ok.task) + .fork(acceptRes.ok.receipt) + } }) } /** - * Schedule Put task to be run by agent. - * A `http/put` task is stored by the service, if it does not exist - * and a receipt is fetched if already available. + * Create allocate and run task if there is no receipt for it already. + * If there is a non expired receipt available, it is returned insted of runing the task again. + * Otherwise, allocation task is executed. * - * @param {object} scheduleAcceptProps - * @param {API.BlobServiceContext} scheduleAcceptProps.context - * @param {API.IssuedInvocationView} scheduleAcceptProps.accept - * @param {API.Invocation} scheduleAcceptProps.acceptfx + * @param {object} allocate + * @param {API.BlobServiceContext} allocate.context + * @param {API.BlobModel} allocate.blob + * @param {API.DIDKey} allocate.space + * @param {API.Link} allocate.cause */ -async function scheduleAccept({ context, accept, acceptfx }) { - let blobAcceptReceipt - - // Get receipt for `blob/accept` if available, otherwise schedule invocation - const receiptGet = await context.receiptsStorage.get(acceptfx.link()) - if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { - return { - error: receiptGet.error, - } - } else if (receiptGet.ok) { - blobAcceptReceipt = receiptGet.ok - } - - // if not already accepted schedule `blob/accept` - if (!blobAcceptReceipt) { - // Execute accept invocation - const acceptRes = await accept.execute(context.getServiceConnection()) - if (acceptRes.out.error) { - return { - error: new AwaitError({ - cause: acceptRes.out.error, - at: 'ucan/wait', - reference: ['.out.ok', acceptfx.cid], - }), - } - } - blobAcceptReceipt = acceptRes - } - - // Create `blob/accept` receipt as conclude invocation to inline as effect - const concludeAccept = createConcludeInvocation( - context.id, - context.id, - blobAcceptReceipt - ) - return { - ok: { - concludeAcceptfx: await concludeAccept.delegate(), +async function allocate({ context, blob, space, cause }) { + // 1. Create web3.storage/blob/allocate invocation and task + const allocate = W3sBlob.allocate.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + blob, + cause: cause, + space, }, - } -} - -/** - * Schedule Put task to be run by agent. - * A `http/put` task is stored by the service, if it does not exist - * and a receipt is fetched if already available. - * - * @param {object} schedulePutProps - * @param {API.BlobServiceContext} schedulePutProps.context - * @param {API.Invocation} schedulePutProps.putfx - */ -async function schedulePut({ context, putfx }) { - // Get receipt for `http/put` if available - const receiptGet = await context.receiptsStorage.get(putfx.link()) - if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { - return { - error: receiptGet.error, - } - } else if (receiptGet.ok) { - // Create `blob/allocate` receipt as conclude invocation to inline as effect - const concludePut = createConcludeInvocation( - context.id, - context.id, - receiptGet.ok - ) - return { - ok: { - concludePutfx: await concludePut.delegate(), - }, - } - } - - // store `http/put` invocation - const invocationPutRes = await context.tasksStorage.put(putfx) - if (invocationPutRes.error) { - // TODO: If already available, do not error? - return { - error: invocationPutRes.error, - } - } - - // TODO: store implementation - // const archiveDelegationRes = await putfx.archive() - // if (archiveDelegationRes.error) { - // return { - // error: archiveDelegationRes.error - // } - // } - - return { - ok: {}, - } -} + expiration: Infinity, + }) + const task = await allocate.delegate() -/** - * Schedule allocate task to be run. - * If there is a non expired receipt, it is returned insted of runing the task again. - * Otherwise, allocation task is scheduled. - * - * @param {object} scheduleAllocateProps - * @param {API.BlobServiceContext} scheduleAllocateProps.context - * @param {API.BlobModel} scheduleAllocateProps.blob - * @param {API.IssuedInvocationView} scheduleAllocateProps.allocate - * @param {API.Invocation} scheduleAllocateProps.allocatefx - */ -async function scheduleAllocate({ context, blob, allocate, allocatefx }) { /** @type {import('@ucanto/interface').Receipt | undefined} */ let blobAllocateReceipt - // Get receipt for `blob/allocate` if available, otherwise schedule invocation - const receiptGet = await context.receiptsStorage.get(allocatefx.link()) + // 2. Get receipt for `blob/allocate` if available, otherwise schedule invocation + const receiptGet = await context.receiptsStorage.get(task.link()) if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { return { error: receiptGet.error, @@ -263,7 +155,7 @@ async function scheduleAllocate({ context, blob, allocate, allocatefx }) { } } - // if not already allocated (or expired) schedule `blob/allocate` + // 3. if not already allocated (or expired) schedule `blob/allocate` if (!blobAllocateReceipt) { // Execute allocate invocation const allocateRes = await allocate.execute(context.getServiceConnection()) @@ -272,51 +164,40 @@ async function scheduleAllocate({ context, blob, allocate, allocatefx }) { error: new AwaitError({ cause: allocateRes.out.error, at: 'ucan/wait', - reference: ['.out.ok', allocatefx.cid], + reference: ['.out.ok', task.link()], }), } } blobAllocateReceipt = allocateRes } - // Create `blob/allocate` receipt as conclude invocation to inline as effect + // 4. Create `blob/allocate` receipt as conclude invocation to inline as effect const concludeAllocate = createConcludeInvocation( context.id, context.id, blobAllocateReceipt ) + return { ok: { - concludeAllocatefx: await concludeAllocate.delegate(), + task, + receipt: await concludeAllocate.delegate() }, } } /** - * Create `blob/add` next tasks. + * Create put task and check if there is a receipt for it already. + * A `http/put` should be task is stored by the service, if it does not exist + * and a receipt is fetched if already available. * - * @param {object} nextProps - * @param {API.BlobServiceContext} nextProps.context - * @param {API.BlobModel} nextProps.blob - * @param {API.DIDKey} nextProps.space - * @param {API.Link} nextProps.cause + * @param {object} put + * @param {API.BlobServiceContext} put.context + * @param {API.BlobModel} put.blob + * @param {API.Invocation} put.allocateTask */ -async function createNextTasks({ context, blob, space, cause }) { - // 1. Create web3.storage/blob/allocate invocation and task - const allocate = W3sBlob.allocate.invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - blob, - cause: cause, - space, - }, - expiration: Infinity, - }) - const allocatefx = await allocate.delegate() - - // 2. Create http/put invocation ans task +async function put({ context, blob, allocateTask }) { + // 1. Create http/put invocation as task // We derive principal from the blob multihash to be an audience // of the `http/put` invocation. That way anyone with blob digest @@ -337,18 +218,76 @@ async function createNextTasks({ context, blob, space, cause }) { nb: { body: blob, url: { - 'ucan/await': ['.out.ok.address.url', allocatefx.cid], + 'ucan/await': ['.out.ok.address.url', allocateTask.link()], }, headers: { - 'ucan/await': ['.out.ok.address.headers', allocatefx.cid], + 'ucan/await': ['.out.ok.address.headers', allocateTask.link()], }, }, facts, expiration: Infinity, }) - const putfx = await put.delegate() + const task = await put.delegate() + + // 2. Get receipt for `http/put` if available + const receiptGet = await context.receiptsStorage.get(task.link()) + if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { + return { + error: receiptGet.error, + } + } else if (receiptGet.ok) { + // 3. Create `blob/allocate` receipt as conclude invocation to inline as effect + const concludePut = createConcludeInvocation( + context.id, + context.id, + receiptGet.ok + ) + return { + ok: { + task, + receipt: await concludePut.delegate(), + }, + } + } + + // 3. store `http/put` invocation + const invocationPutRes = await context.tasksStorage.put(task) + if (invocationPutRes.error) { + // TODO: If already available, do not error? + return { + error: invocationPutRes.error, + } + } - // 3. Create web3.storage/blob/accept invocation and task + // TODO: store implementation + // const archiveDelegationRes = await task.archive() + // if (archiveDelegationRes.error) { + // return { + // error: archiveDelegationRes.error + // } + // } + + return { + ok: { + task, + receipt: undefined + }, + } +} + +/** + * Create accept and run task if there is no receipt. + * A accept task can run when `http/put` receipt already exists. + * + * @param {object} accept + * @param {API.BlobServiceContext} accept.context + * @param {API.BlobModel} accept.blob + * @param {API.DIDKey} accept.space + * @param {API.Invocation} accept.putTask + * @param {API.Invocation} [accept.putReceipt] + */ +async function accept({ context, blob, space, putTask, putReceipt }) { + // 1. Create web3.storage/blob/accept invocation and task const accept = W3sBlob.accept.invoke({ issuer: context.id, audience: context.id, @@ -356,18 +295,59 @@ async function createNextTasks({ context, blob, space, cause }) { nb: { blob, space, - _put: { 'ucan/await': ['.out.ok', putfx.link()] }, + _put: { 'ucan/await': ['.out.ok', putTask.link()] }, }, expiration: Infinity, }) - const acceptfx = await accept.delegate() + const task = await accept.delegate() + // 2. If there is not put receipt, `accept` is still blocked + if (!putReceipt) { + return { + ok: { + task, + receipt: undefined + } + } + } + + // 3. Get receipt for `blob/accept` if available, otherwise schedule invocation + let blobAcceptReceipt + const receiptGet = await context.receiptsStorage.get(task.link()) + if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { + return { + error: receiptGet.error, + } + } else if (receiptGet.ok) { + blobAcceptReceipt = receiptGet.ok + } + + // 4. if not already accepted schedule `blob/accept` + if (!blobAcceptReceipt) { + // Execute accept invocation + const acceptRes = await accept.execute(context.getServiceConnection()) + if (acceptRes.out.error) { + return { + error: new AwaitError({ + cause: acceptRes.out.error, + at: 'ucan/wait', + reference: ['.out.ok', task.link()], + }), + } + } + blobAcceptReceipt = acceptRes + } + + // Create `blob/accept` receipt as conclude invocation to inline as effect + const concludeAccept = createConcludeInvocation( + context.id, + context.id, + blobAcceptReceipt + ) return { - allocate, - allocatefx, - put, - putfx, - accept, - acceptfx, + ok: { + task, + receipt: await concludeAccept.delegate(), + }, } } diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 87d1227fb..ea153fe3d 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -60,24 +60,23 @@ export const test = { // validate receipt next const next = parseBlobAddReceiptNext(blobAdd) - assert.ok(next.allocatefx) - assert.ok(next.putfx) - assert.ok(next.acceptfx) - assert.equal(next.concludefxs.length, 1) - assert.ok(next.allocateReceipt) - assert.ok(!next.putReceipt) - assert.ok(!next.acceptReceipt) + assert.ok(next.allocate.task) + assert.ok(next.put.task) + assert.ok(next.accept.task) + assert.ok(next.allocate.receipt) + assert.ok(!next.put.receipt) + assert.ok(!next.accept.receipt) // validate facts exist for `http/put` - assert.ok(next.putfx.facts.length) - assert.ok(next.putfx.facts[0]['keys']) + assert.ok(next.put.task.facts.length) + assert.ok(next.put.task.facts[0]['keys']) // Validate `http/put` invocation was stored - const httpPutGetTask = await context.tasksStorage.get(next.putfx.cid) + const httpPutGetTask = await context.tasksStorage.get(next.put.task.cid) assert.ok(httpPutGetTask.ok) // validate that scheduled allocate task executed and has its receipt content - const receipt = next.allocateReceipt + const receipt = next.allocate.receipt assert.ok(receipt.out) assert.ok(receipt.out.ok) assert.equal(receipt.out.ok?.size, size) @@ -122,18 +121,17 @@ export const test = { // parse first receipt next const firstNext = parseBlobAddReceiptNext(firstBlobAdd) - assert.ok(firstNext.allocatefx) - assert.ok(firstNext.putfx) - assert.ok(firstNext.acceptfx) - assert.equal(firstNext.concludefxs.length, 1) - assert.ok(firstNext.allocateReceipt) - assert.ok(!firstNext.putReceipt) - assert.ok(!firstNext.acceptReceipt) + assert.ok(firstNext.allocate.task) + assert.ok(firstNext.put.task) + assert.ok(firstNext.accept.task) + assert.ok(firstNext.allocate.receipt) + assert.ok(!firstNext.put.receipt) + assert.ok(!firstNext.accept.receipt) // Store allocate receipt to not re-schedule const receiptPutRes = await context.receiptsStorage.put( // @ts-expect-error types unknown for next - firstNext.allocateReceipt + firstNext.allocate.receipt ) assert.ok(receiptPutRes.ok) @@ -145,16 +143,15 @@ export const test = { // parse second receipt next const secondNext = parseBlobAddReceiptNext(secondBlobAdd) - assert.ok(secondNext.allocatefx) - assert.ok(secondNext.putfx) - assert.ok(secondNext.acceptfx) - assert.equal(secondNext.concludefxs.length, 1) - assert.ok(secondNext.allocateReceipt) - assert.ok(!secondNext.putReceipt) - assert.ok(!secondNext.acceptReceipt) + assert.ok(secondNext.allocate.task) + assert.ok(secondNext.put.task) + assert.ok(secondNext.accept.task) + assert.ok(secondNext.allocate.receipt) + assert.ok(!secondNext.put.receipt) + assert.ok(!secondNext.accept.receipt) // allocate receipt is from same invocation CID assert.ok( - firstNext.concludefxs[0].cid.equals(secondNext.concludefxs[0].cid) + firstNext.allocate.task.link().equals(secondNext.allocate.task.link()) ) }, 'blob/add schedules allocation and returns effects for allocate, accept and put together with their receipts (when stored)': @@ -194,24 +191,23 @@ export const test = { // parse first receipt next const firstNext = parseBlobAddReceiptNext(firstBlobAdd) - assert.ok(firstNext.allocatefx) - assert.ok(firstNext.putfx) - assert.ok(firstNext.acceptfx) - assert.equal(firstNext.concludefxs.length, 1) - assert.ok(firstNext.allocateReceipt) - assert.ok(!firstNext.putReceipt) - assert.ok(!firstNext.acceptReceipt) + assert.ok(firstNext.allocate.task) + assert.ok(firstNext.put.task) + assert.ok(firstNext.accept.task) + assert.ok(firstNext.allocate.receipt) + assert.ok(!firstNext.put.receipt) + assert.ok(!firstNext.accept.receipt) // Store allocate receipt to not re-schedule const receiptPutRes = await context.receiptsStorage.put( // @ts-expect-error types unknown for next - firstNext.allocateReceipt + firstNext.allocate.receipt ) assert.ok(receiptPutRes.ok) /** @type {import('@web3-storage/capabilities/types').BlobAddress} */ // @ts-expect-error receipt type is unknown - const address = firstNext.allocateReceipt.out.ok.address + const address = firstNext.allocate.receipt.out.ok.address // Store the blob to the address const goodPut = await fetch(address.url, { @@ -230,20 +226,19 @@ export const test = { // parse second receipt next const secondNext = parseBlobAddReceiptNext(secondBlobAdd) - assert.ok(secondNext.allocatefx) - assert.ok(secondNext.putfx) - assert.ok(secondNext.acceptfx) - assert.equal(secondNext.concludefxs.length, 1) - assert.ok(secondNext.allocateReceipt) - assert.ok(!secondNext.putReceipt) - assert.ok(!secondNext.acceptReceipt) + assert.ok(secondNext.allocate.task) + assert.ok(secondNext.put.task) + assert.ok(secondNext.accept.task) + assert.ok(secondNext.allocate.receipt) + assert.ok(!secondNext.put.receipt) + assert.ok(!secondNext.accept.receipt) // Store blob/allocate given conclude needs it to schedule blob/accept // Store allocate task to be fetchable from allocate - await context.tasksStorage.put(secondNext.allocatefx) + await context.tasksStorage.put(secondNext.allocate.task) // Invoke `conclude` with `http/put` receipt - const keys = secondNext.putfx.facts[0]['keys'] + const keys = secondNext.put.task.facts[0]['keys'] // @ts-expect-error Argument of type 'unknown' is not assignable to parameter of type 'SignerArchive<`did:${string}:${string}`, SigAlg>' const blobProvider = ed25519.from(keys) const httpPut = HTTPCapabilities.put.invoke({ @@ -256,16 +251,16 @@ export const test = { size, }, url: { - 'ucan/await': ['.out.ok.address.url', secondNext.allocatefx.cid], + 'ucan/await': ['.out.ok.address.url', secondNext.allocate.task.cid], }, headers: { 'ucan/await': [ '.out.ok.address.headers', - secondNext.allocatefx.cid, + secondNext.allocate.task.cid, ], }, }, - facts: secondNext.putfx.facts, + facts: secondNext.put.task.facts, expiration: Infinity, }) @@ -284,7 +279,6 @@ export const test = { ) const ucanConclude = await httpPutConcludeInvocation.execute(connection) if (!ucanConclude.out.ok) { - console.log('ucan conclude', ucanConclude.out.error) throw new Error('invocation failed', { cause: ucanConclude.out }) } @@ -296,17 +290,16 @@ export const test = { // parse third receipt next const thirdNext = parseBlobAddReceiptNext(thirdBlobAdd) - assert.ok(thirdNext.allocatefx) - assert.ok(thirdNext.putfx) - assert.ok(thirdNext.acceptfx) - assert.equal(thirdNext.concludefxs.length, 3) - assert.ok(thirdNext.allocateReceipt) - assert.ok(thirdNext.putReceipt) - assert.ok(thirdNext.acceptReceipt) - - assert.ok(thirdNext.allocateReceipt.out.ok?.address) - assert.deepEqual(thirdNext.putReceipt?.out.ok, {}) - assert.ok(thirdNext.acceptReceipt?.out.ok?.site) + assert.ok(thirdNext.allocate.task) + assert.ok(thirdNext.put.task) + assert.ok(thirdNext.accept.task) + assert.ok(thirdNext.allocate.receipt) + assert.ok(thirdNext.put.receipt) + assert.ok(thirdNext.accept.receipt) + + assert.ok(thirdNext.allocate.receipt.out.ok?.address) + assert.deepEqual(thirdNext.put.receipt?.out.ok, {}) + assert.ok(thirdNext.accept.receipt?.out.ok?.site) }, 'blob/add fails when a blob with size bigger than maximum size is added': async (assert, context) => { diff --git a/packages/upload-api/test/helpers/blob.js b/packages/upload-api/test/helpers/blob.js index 4cc710108..369150c84 100644 --- a/packages/upload-api/test/helpers/blob.js +++ b/packages/upload-api/test/helpers/blob.js @@ -16,17 +16,17 @@ export function parseBlobAddReceiptNext(receipt) { **/ // @ts-expect-error read only effect const forkInvocations = receipt.fx.fork - const allocatefx = forkInvocations.find( + const allocateTask = forkInvocations.find( (fork) => fork.capabilities[0].can === W3sBlobCapabilities.allocate.can ) const concludefxs = forkInvocations.filter( (fork) => fork.capabilities[0].can === UCAN.conclude.can ) - const putfx = forkInvocations.find( + const putTask = forkInvocations.find( (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can ) - const acceptfx = receipt.fx.join - if (!allocatefx || !concludefxs.length || !putfx || !acceptfx) { + const acceptTask = receipt.fx.join + if (!allocateTask || !concludefxs.length || !putTask || !acceptTask) { throw new Error('mandatory effects not received') } @@ -35,17 +35,17 @@ export function parseBlobAddReceiptNext(receipt) { /** @type {API.Receipt | undefined} */ // @ts-expect-error types unknown for next const allocateReceipt = nextReceipts.find((receipt) => - receipt.ran.link().equals(allocatefx.cid) + receipt.ran.link().equals(allocateTask.cid) ) /** @type {API.Receipt<{}, API.Failure> | undefined} */ // @ts-expect-error types unknown for next const putReceipt = nextReceipts.find((receipt) => - receipt.ran.link().equals(putfx.cid) + receipt.ran.link().equals(putTask.cid) ) /** @type {API.Receipt | undefined} */ // @ts-expect-error types unknown for next const acceptReceipt = nextReceipts.find((receipt) => - receipt.ran.link().equals(acceptfx.link()) + receipt.ran.link().equals(acceptTask.link()) ) if (!allocateReceipt) { @@ -53,12 +53,17 @@ export function parseBlobAddReceiptNext(receipt) { } return { - allocatefx, - allocateReceipt, - concludefxs, - putfx, - putReceipt, - acceptfx, - acceptReceipt, + allocate: { + task: allocateTask, + receipt: allocateReceipt, + }, + put: { + task: putTask, + receipt: putReceipt + }, + accept: { + task: acceptTask, + receipt: acceptReceipt + } } } From b91dd046c6670dfac886333a967071013c7c7a1b Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 11 Apr 2024 15:11:23 +0200 Subject: [PATCH 24/27] test: simplify and add tests --- packages/upload-api/src/blob/accept.js | 29 +-- packages/upload-api/src/blob/add.js | 47 ++--- packages/upload-api/src/blob/lib.js | 1 - packages/upload-api/src/ucan/conclude.js | 4 +- packages/upload-api/src/ucan/lib.js | 3 +- packages/upload-api/test/handlers/ucan.js | 91 +++++++--- .../upload-api/test/handlers/web3.storage.js | 165 +++++++++++++++++- packages/upload-api/test/helpers/blob.js | 6 +- 8 files changed, 273 insertions(+), 73 deletions(-) diff --git a/packages/upload-api/src/blob/accept.js b/packages/upload-api/src/blob/accept.js index 9dc32bbff..13cee2868 100644 --- a/packages/upload-api/src/blob/accept.js +++ b/packages/upload-api/src/blob/accept.js @@ -24,6 +24,8 @@ export function blobAcceptProvider(context) { // If blob is not stored, we must fail const hasBlob = await context.blobsStorage.has(blob.digest) if (hasBlob.error) { + return hasBlob + } else if (!hasBlob.ok) { return { error: new AllocatedMemoryHadNotBeenWrittenTo(), } @@ -34,20 +36,19 @@ export function blobAcceptProvider(context) { const content = createLink(rawCode, digest) const w3link = `https://w3s.link/ipfs/${content.toString()}?origin=r2://${R2_REGION}/${R2_BUCKET}` - const locationClaim = await Assert.location - .delegate({ - issuer: context.id, - audience: DID.parse(space), - with: context.id.toDIDKey(), - nb: { - content, - location: [ - // @ts-expect-error Type 'string' is not assignable to type '`${string}:${string}`' - w3link, - ], - }, - expiration: Infinity, - }) + const locationClaim = await Assert.location.delegate({ + issuer: context.id, + audience: DID.parse(space), + with: context.id.toDIDKey(), + nb: { + content, + location: [ + // @ts-expect-error Type 'string' is not assignable to type '`${string}:${string}`' + w3link, + ], + }, + expiration: Infinity, + }) // Create result object /** @type {API.OkBuilder} */ diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 117271f33..e55f5dba0 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -23,7 +23,10 @@ export function blobAddProvider(context) { // Verify blob is within accept size if (blob.size > context.maxUploadSize) { return { - error: new BlobSizeOutsideOfSupportedRange(blob.size, context.maxUploadSize), + error: new BlobSizeOutsideOfSupportedRange( + blob.size, + context.maxUploadSize + ), } } @@ -42,7 +45,7 @@ export function blobAddProvider(context) { const putRes = await put({ context, blob, - allocateTask: allocateRes.ok.task + allocateTask: allocateRes.ok.task, }) if (putRes.error) { return putRes @@ -54,7 +57,7 @@ export function blobAddProvider(context) { blob, space, putTask: putRes.ok.task, - putReceipt: putRes.ok.receipt + putReceipt: putRes.ok.receipt, }) if (acceptRes.error) { return acceptRes @@ -85,20 +88,22 @@ export function blobAddProvider(context) { ) } - return result - // 1. System attempts to allocate memory in user space for the blob. - .fork(allocateRes.ok.task) - .fork(allocateRes.ok.receipt) - // 2. System requests user agent (or anyone really) to upload the content - // corresponding to the blob - // via HTTP PUT to given location. - .fork(putRes.ok.task) - .fork(putRes.ok.receipt) - // 3. System will attempt to accept uploaded content that matches blob - // multihash and size. - .join(acceptRes.ok.task) - .fork(acceptRes.ok.receipt) - } + return ( + result + // 1. System attempts to allocate memory in user space for the blob. + .fork(allocateRes.ok.task) + .fork(allocateRes.ok.receipt) + // 2. System requests user agent (or anyone really) to upload the content + // corresponding to the blob + // via HTTP PUT to given location. + .fork(putRes.ok.task) + .fork(putRes.ok.receipt) + // 3. System will attempt to accept uploaded content that matches blob + // multihash and size. + .join(acceptRes.ok.task) + .fork(acceptRes.ok.receipt) + ) + }, }) } @@ -181,7 +186,7 @@ async function allocate({ context, blob, space, cause }) { return { ok: { task, - receipt: await concludeAllocate.delegate() + receipt: await concludeAllocate.delegate(), }, } } @@ -270,7 +275,7 @@ async function put({ context, blob, allocateTask }) { return { ok: { task, - receipt: undefined + receipt: undefined, }, } } @@ -306,8 +311,8 @@ async function accept({ context, blob, space, putTask, putReceipt }) { return { ok: { task, - receipt: undefined - } + receipt: undefined, + }, } } diff --git a/packages/upload-api/src/blob/lib.js b/packages/upload-api/src/blob/lib.js index 556c74176..59f939794 100644 --- a/packages/upload-api/src/blob/lib.js +++ b/packages/upload-api/src/blob/lib.js @@ -3,7 +3,6 @@ import { Failure } from '@ucanto/server' export const AllocatedMemoryHadNotBeenWrittenToName = 'AllocatedMemoryHadNotBeenWrittenTo' export class AllocatedMemoryHadNotBeenWrittenTo extends Failure { - get name() { return AllocatedMemoryHadNotBeenWrittenToName } diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index e6dd54046..dcc49cb2d 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -27,7 +27,7 @@ export const ucanConcludeProvider = ({ if (httpPutTaskGetRes.error) { if (httpPutTaskGetRes.error.name === 'RecordNotFound') { return { - error: new ReferencedInvocationNotFound(ranInvocation.link()) + error: new ReferencedInvocationNotFound(ranInvocation.link()), } } return httpPutTaskGetRes @@ -53,7 +53,7 @@ export const ucanConcludeProvider = ({ // Get triggering task (blob/allocate) by checking blocking task from `url` /** @type {API.UnknownLink} */ // @ts-expect-error ts does not know how to get this - const [,blobAllocateTaskCid] = cap.nb.url['ucan/await'] + const [, blobAllocateTaskCid] = cap.nb.url['ucan/await'] const blobAllocateTaskGet = await tasksStorage.get( blobAllocateTaskCid ) diff --git a/packages/upload-api/src/ucan/lib.js b/packages/upload-api/src/ucan/lib.js index 11b05fc8e..7d5d33c43 100644 --- a/packages/upload-api/src/ucan/lib.js +++ b/packages/upload-api/src/ucan/lib.js @@ -1,7 +1,6 @@ import { Failure } from '@ucanto/server' -export const ReferencedInvocationNotFoundName = - 'ReferencedInvocationNotFound' +export const ReferencedInvocationNotFoundName = 'ReferencedInvocationNotFound' export class ReferencedInvocationNotFound extends Failure { /** * @param {import('@ucanto/interface').Link} [invocation] diff --git a/packages/upload-api/test/handlers/ucan.js b/packages/upload-api/test/handlers/ucan.js index 7758519f4..0a586db98 100644 --- a/packages/upload-api/test/handlers/ucan.js +++ b/packages/upload-api/test/handlers/ucan.js @@ -10,10 +10,9 @@ import * as HTTPCapabilities from '@web3-storage/capabilities/http' import { createServer, connect } from '../../src/lib.js' import { alice, bob, mallory, registerSpace } from '../util.js' -import { - createConcludeInvocation, - getConcludeReceipt, -} from '../../src/ucan/conclude.js' +import { createConcludeInvocation } from '../../src/ucan/conclude.js' +import { ReferencedInvocationNotFoundName } from '../../src/ucan/lib.js' +import { parseBlobAddReceiptNext } from '../helpers/blob.js' /** * @type {API.Tests} @@ -365,6 +364,55 @@ export const test = { assert.ok(String(revoke.out.error?.message).match(/Constrain violation/)) }, + 'ucan/conclude writes a receipt for a task previously scheduled': async ( + assert, + context + ) => { + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + // Invoke something + const proof = await Console.log.delegate({ + issuer: context.id, + audience: alice, + with: context.id.did(), + }) + const invocation = Console.log.invoke({ + issuer: alice, + audience: context.id, + with: context.id.did(), + nb: { value: 'hello' }, + proofs: [proof], + }) + + const success = await invocation.execute(connection) + + if (!success.out.ok) { + throw new Error('invocation failed', { cause: success }) + } + + // Create conclude invocation + const concludeInvocation = createConcludeInvocation( + alice, + context.id, + success + ) + const ucanConcludeFail = await concludeInvocation.execute(connection) + assert.ok(ucanConcludeFail.out.error) + assert.equal( + ucanConcludeFail.out.error?.name, + ReferencedInvocationNotFoundName + ) + + // Store scheduled task + await context.tasksStorage.put(await invocation.delegate()) + + const ucanConcludeSuccess = await concludeInvocation.execute(connection) + assert.ok(ucanConcludeSuccess.out.ok) + assert.ok(ucanConcludeSuccess.out.ok?.time) + }, 'ucan/conclude schedules web3.storage/blob/accept if invoked with the blob put receipt': async (assert, context) => { const taskScheduled = pDefer() @@ -412,35 +460,17 @@ export const test = { } // Get receipt relevant content - /** - * @type {import('@ucanto/interface').Invocation[]} - **/ - // @ts-expect-error read only effect - const forkInvocations = blobAdd.fx.fork - const allocatefx = forkInvocations.find( - (fork) => fork.capabilities[0].can === W3sBlobCapabilities.allocate.can - ) - const allocateUcanConcludefx = forkInvocations.find( - (fork) => fork.capabilities[0].can === UCAN.conclude.can - ) - const putfx = forkInvocations.find( - (fork) => fork.capabilities[0].can === HTTPCapabilities.put.can - ) - if (!allocateUcanConcludefx || !putfx || !allocatefx) { - throw new Error('effects not provided') - } - const receipt = getConcludeReceipt(allocateUcanConcludefx) + const next = parseBlobAddReceiptNext(blobAdd) - // Get `web3.storage/blob/allocate` receipt with address /** * @type {import('@web3-storage/capabilities/types').BlobAddress} **/ // @ts-expect-error receipt out is unknown - const address = receipt?.out.ok?.address + const address = next.allocate.receipt.out.ok?.address assert.ok(address) // Store allocate task to be fetchable from allocate - await context.tasksStorage.put(allocatefx) + await context.tasksStorage.put(next.allocate.task) // Write blob const goodPut = await fetch(address.url, { @@ -452,7 +482,7 @@ export const test = { assert.equal(goodPut.status, 200, await goodPut.text()) // Create `http/put` receipt - const keys = putfx.facts[0]['keys'] + const keys = next.put.task.facts[0]['keys'] // @ts-expect-error Argument of type 'unknown' is not assignable to parameter of type 'SignerArchive<`did:${string}:${string}`, SigAlg>' const blobProvider = ed25519.from(keys) const httpPut = HTTPCapabilities.put.invoke({ @@ -465,13 +495,16 @@ export const test = { size, }, url: { - 'ucan/await': ['.out.ok.address.url', allocatefx.cid], + 'ucan/await': ['.out.ok.address.url', next.allocate.task.link()], }, headers: { - 'ucan/await': ['.out.ok.address.headers', allocatefx.cid], + 'ucan/await': [ + '.out.ok.address.headers', + next.allocate.task.link(), + ], }, }, - facts: putfx.facts, + facts: next.put.task.facts, expiration: Infinity, }) diff --git a/packages/upload-api/test/handlers/web3.storage.js b/packages/upload-api/test/handlers/web3.storage.js index 1a9b8945e..a527e3707 100644 --- a/packages/upload-api/test/handlers/web3.storage.js +++ b/packages/upload-api/test/handlers/web3.storage.js @@ -1,14 +1,20 @@ import * as API from '../../src/types.js' import { equals } from 'uint8arrays' +import { create as createLink } from 'multiformats/link' import { Absentee } from '@ucanto/principal' +import { Digest } from 'multiformats/hashes/digest' import { sha256 } from 'multiformats/hashes/sha2' +import { code as rawCode } from 'multiformats/codecs/raw' +import { Assert } from '@web3-storage/content-claims/capability' import * as BlobCapabilities from '@web3-storage/capabilities/blob' import * as W3sBlobCapabilities from '@web3-storage/capabilities/web3.storage/blob' import { base64pad } from 'multiformats/bases/base64' +import { AllocatedMemoryHadNotBeenWrittenToName } from '../../src/blob/lib.js' import { provisionProvider } from '../helpers/utils.js' import { createServer, connect } from '../../src/lib.js' import { alice, bob, createSpace, registerSpace } from '../util.js' +import { parseBlobAddReceiptNext } from '../helpers/blob.js' /** * @type {API.Tests} @@ -69,6 +75,7 @@ export const test = { assert.ok(blobAllocate.out.ok.address) assert.ok(blobAllocate.out.ok.address?.headers) assert.ok(blobAllocate.out.ok.address?.url) + assert.ok(blobAllocate.out.ok.address?.expiresAt) assert.equal( blobAllocate.out.ok.address?.headers?.['content-length'], String(size) @@ -513,5 +520,161 @@ export const test = { const retryBlobAllocate = await serviceBlobAllocate.execute(connection) assert.equal(retryBlobAllocate.out.error, undefined) }, - // TODO: Blob accept + 'web3.storage/blob/accept returns site delegation': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + const content = createLink( + rawCode, + new Digest(sha256.code, 32, digest, digest) + ) + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const blobAdd = await blobAddInvocation.execute(connection) + if (!blobAdd.out.ok) { + throw new Error('invocation failed', { cause: blobAdd }) + } + + // parse receipt next + const next = parseBlobAddReceiptNext(blobAdd) + + /** @type {import('@web3-storage/capabilities/types').BlobAddress} */ + // @ts-expect-error receipt type is unknown + const address = next.allocate.receipt.out.ok.address + + // Store the blob to the address + const goodPut = await fetch(address.url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: address.headers, + }) + assert.equal(goodPut.status, 200, await goodPut.text()) + + // invoke `web3.storage/blob/accept` + const serviceBlobAccept = W3sBlobCapabilities.accept.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + blob: { + digest, + size, + }, + space: spaceDid, + _put: { 'ucan/await': ['.out.ok', next.put.task.link()] }, + }, + proofs: [proof], + }) + const blobAccept = await serviceBlobAccept.execute(connection) + if (!blobAccept.out.ok) { + throw new Error('invocation failed', { cause: blobAccept }) + } + // Validate out + assert.ok(blobAccept.out.ok) + assert.ok(blobAccept.out.ok.site) + + // Validate effect + assert.equal(blobAccept.fx.fork.length, 1) + /** @type {import('@ucanto/interface').Delegation} */ + // @ts-expect-error delegation not assignable to Effect per TS understanding + const delegation = blobAccept.fx.fork[0] + assert.equal(delegation.capabilities.length, 1) + assert.ok(delegation.capabilities[0].can, Assert.location.can) + // @ts-expect-error nb unknown + assert.ok(delegation.capabilities[0].nb.content.equals(content)) + // @ts-expect-error nb unknown + const locations = delegation.capabilities[0].nb.location + assert.equal(locations.length, 1) + assert.ok( + locations[0].includes( + `https://w3s.link/ipfs/${content.toString()}?origin` + ) + ) + }, + 'web3.storage/blob/accept fails to provide site delegation when blob was not stored': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const blobAdd = await blobAddInvocation.execute(connection) + if (!blobAdd.out.ok) { + throw new Error('invocation failed', { cause: blobAdd }) + } + + // parse receipt next + const next = parseBlobAddReceiptNext(blobAdd) + + // invoke `web3.storage/blob/accept` + const serviceBlobAccept = W3sBlobCapabilities.accept.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + blob: { + digest, + size, + }, + space: spaceDid, + _put: { 'ucan/await': ['.out.ok', next.put.task.link()] }, + }, + proofs: [proof], + }) + const blobAccept = await serviceBlobAccept.execute(connection) + // Validate out error + assert.ok(blobAccept.out.error) + assert.equal( + blobAccept.out.error?.name, + AllocatedMemoryHadNotBeenWrittenToName + ) + }, } diff --git a/packages/upload-api/test/helpers/blob.js b/packages/upload-api/test/helpers/blob.js index 369150c84..1bd2a36d6 100644 --- a/packages/upload-api/test/helpers/blob.js +++ b/packages/upload-api/test/helpers/blob.js @@ -59,11 +59,11 @@ export function parseBlobAddReceiptNext(receipt) { }, put: { task: putTask, - receipt: putReceipt + receipt: putReceipt, }, accept: { task: acceptTask, - receipt: acceptReceipt - } + receipt: acceptReceipt, + }, } } From 8b0d4ab4c4211d528bb4b9a9d442876f953e861d Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 11 Apr 2024 18:06:28 +0200 Subject: [PATCH 25/27] test: add all storage tests --- packages/upload-api/src/blob/add.js | 10 - packages/upload-api/src/errors.js | 14 +- packages/upload-api/src/types.ts | 1 + packages/upload-api/src/types/service.ts | 3 +- packages/upload-api/test/handlers/blob.js | 2 - .../test/storage/allocations-storage-tests.js | 313 ++++++++++++++++++ .../test/storage/allocations-storage.js | 5 +- .../test/storage/allocations-storage.spec.js | 3 + .../test/storage/blobs-storage-tests.js | 46 +++ .../test/storage/blobs-storage.spec.js | 3 + .../test/storage/receipts-storage-tests.js | 108 ++++++ .../test/storage/receipts-storage.spec.js | 3 + .../test/storage/tasks-storage-tests.js | 95 ++++++ .../upload-api/test/storage/tasks-storage.js | 10 +- .../test/storage/tasks-storage.spec.js | 3 + 15 files changed, 595 insertions(+), 24 deletions(-) create mode 100644 packages/upload-api/test/storage/allocations-storage-tests.js create mode 100644 packages/upload-api/test/storage/allocations-storage.spec.js create mode 100644 packages/upload-api/test/storage/blobs-storage-tests.js create mode 100644 packages/upload-api/test/storage/blobs-storage.spec.js create mode 100644 packages/upload-api/test/storage/receipts-storage-tests.js create mode 100644 packages/upload-api/test/storage/receipts-storage.spec.js create mode 100644 packages/upload-api/test/storage/tasks-storage-tests.js create mode 100644 packages/upload-api/test/storage/tasks-storage.spec.js diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index e55f5dba0..17896ab67 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -143,7 +143,6 @@ async function allocate({ context, blob, space, cause }) { error: receiptGet.error, } } else if (receiptGet.ok) { - // @ts-expect-error ts not able to cast receipt blobAllocateReceipt = receiptGet.ok // Verify if allocation is expired before "accepting" this receipt. @@ -258,20 +257,11 @@ async function put({ context, blob, allocateTask }) { // 3. store `http/put` invocation const invocationPutRes = await context.tasksStorage.put(task) if (invocationPutRes.error) { - // TODO: If already available, do not error? return { error: invocationPutRes.error, } } - // TODO: store implementation - // const archiveDelegationRes = await task.archive() - // if (archiveDelegationRes.error) { - // return { - // error: archiveDelegationRes.error - // } - // } - return { ok: { task, diff --git a/packages/upload-api/src/errors.js b/packages/upload-api/src/errors.js index 885c8a080..9c7df6fe8 100644 --- a/packages/upload-api/src/errors.js +++ b/packages/upload-api/src/errors.js @@ -13,26 +13,24 @@ export class StoreOperationFailed extends Server.Failure { } } -export const RecordNotFoundErrorName = /** @type {const} */ ('RecordNotFound') -export class RecordNotFound extends Server.Failure { +export const RecordKeyConflictName = /** @type {const} */ ('RecordKeyConflict') +export class RecordKeyConflict extends Server.Failure { get reason() { return this.message } get name() { - return RecordNotFoundErrorName + return RecordKeyConflictName } } -export const DecodeBlockOperationErrorName = /** @type {const} */ ( - 'DecodeBlockOperationFailed' -) -export class DecodeBlockOperationFailed extends Server.Failure { +export const RecordNotFoundErrorName = /** @type {const} */ ('RecordNotFound') +export class RecordNotFound extends Server.Failure { get reason() { return this.message } get name() { - return DecodeBlockOperationErrorName + return RecordNotFoundErrorName } } diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 51c78ede6..ff91d076f 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -464,6 +464,7 @@ export interface UcantoServerContext extends ServiceContext, RevocationChecker { export interface UcantoServerTestContext extends UcantoServerContext, StoreTestContext, + BlobServiceContext, UploadTestContext { connection: ConnectionView mail: DebugEmail diff --git a/packages/upload-api/src/types/service.ts b/packages/upload-api/src/types/service.ts index 6e9858572..2575e5a3a 100644 --- a/packages/upload-api/src/types/service.ts +++ b/packages/upload-api/src/types/service.ts @@ -8,7 +8,8 @@ import type { } from '@ucanto/interface' import { Storage } from './storage.js' -export type ReceiptsStorage = Storage +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type ReceiptsStorage = Storage> export interface TasksScheduler { schedule: (invocation: Invocation) => Promise> } diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index ea153fe3d..ac9eb2366 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -130,7 +130,6 @@ export const test = { // Store allocate receipt to not re-schedule const receiptPutRes = await context.receiptsStorage.put( - // @ts-expect-error types unknown for next firstNext.allocate.receipt ) assert.ok(receiptPutRes.ok) @@ -200,7 +199,6 @@ export const test = { // Store allocate receipt to not re-schedule const receiptPutRes = await context.receiptsStorage.put( - // @ts-expect-error types unknown for next firstNext.allocate.receipt ) assert.ok(receiptPutRes.ok) diff --git a/packages/upload-api/test/storage/allocations-storage-tests.js b/packages/upload-api/test/storage/allocations-storage-tests.js new file mode 100644 index 000000000..cb74574a9 --- /dev/null +++ b/packages/upload-api/test/storage/allocations-storage-tests.js @@ -0,0 +1,313 @@ +import * as API from '../../src/types.js' + +import { sha256 } from 'multiformats/hashes/sha2' +import * as BlobCapabilities from '@web3-storage/capabilities/blob' +import { equals } from 'uint8arrays' + +import { + RecordKeyConflictName, + RecordNotFoundErrorName, +} from '../../src/errors.js' +import { alice, bob, registerSpace } from '../util.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'should store allocations': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const allocationsStorage = context.allocationsStorage + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const invocation = (await blobAdd.delegate()).link() + + const allocationInsert = await allocationsStorage.insert({ + space: spaceDid, + blob: { + digest, + size, + }, + invocation, + }) + + assert.ok(allocationInsert.ok) + assert.ok(allocationInsert.ok?.blob) + }, + 'should store same allocation once': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const allocationsStorage = context.allocationsStorage + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const invocation = (await blobAdd.delegate()).link() + + const allocationInsert0 = await allocationsStorage.insert({ + space: spaceDid, + blob: { + digest, + size, + }, + invocation, + }) + assert.ok(allocationInsert0.ok) + + const allocationInsert1 = await allocationsStorage.insert({ + space: spaceDid, + blob: { + digest, + size, + }, + invocation, + }) + assert.ok(allocationInsert1.error) + assert.equal(allocationInsert1.error?.name, RecordKeyConflictName) + }, + 'should get allocations only when available': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const allocationsStorage = context.allocationsStorage + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + const allocationGet0 = await allocationsStorage.get(spaceDid, digest) + assert.ok(allocationGet0.error) + assert.equal(allocationGet0.error?.name, RecordNotFoundErrorName) + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const invocation = (await blobAdd.delegate()).link() + + const allocationInsert = await allocationsStorage.insert({ + space: spaceDid, + blob: { + digest, + size, + }, + invocation, + }) + + assert.ok(allocationInsert.ok) + assert.ok(allocationInsert.ok?.blob) + + const allocationGet1 = await allocationsStorage.get(spaceDid, digest) + assert.ok(allocationGet1.ok) + assert.ok(allocationGet1.ok?.blob) + assert.equal(allocationGet1.ok?.blob.size, size) + assert.ok( + equals(digest, allocationGet1.ok?.blob.digest || new Uint8Array()) + ) + assert.ok(allocationGet1.ok?.invocation) + }, + 'should verify allocations exist': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const allocationsStorage = context.allocationsStorage + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + const allocationExist0 = await allocationsStorage.exists(spaceDid, digest) + assert.ok(!allocationExist0.error) + assert.ok(!allocationExist0.ok) + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const invocation = (await blobAdd.delegate()).link() + + const allocationInsert = await allocationsStorage.insert({ + space: spaceDid, + blob: { + digest, + size, + }, + invocation, + }) + + assert.ok(allocationInsert.ok) + assert.ok(allocationInsert.ok?.blob) + + const allocationExist1 = await allocationsStorage.exists(spaceDid, digest) + assert.ok(allocationExist1.ok) + assert.ok(!allocationExist1.error) + }, + 'should list all allocations in a space': async (assert, context) => { + const { proof: aliceProof, spaceDid: aliceSpaceDid } = await registerSpace( + alice, + context + ) + const { proof: bobProof, spaceDid: bobSpaceDid } = await registerSpace( + bob, + context + ) + const allocationsStorage = context.allocationsStorage + + // Data for alice + const data0 = new Uint8Array([11, 22, 34, 44, 55]) + const multihash0 = await sha256.digest(data0) + const digest0 = multihash0.bytes + const size0 = data0.byteLength + const blob0 = { + digest: digest0, + size: size0, + } + // Data for bob + const data1 = new Uint8Array([66, 77, 88, 99, 0]) + const multihash1 = await sha256.digest(data1) + const digest1 = multihash1.bytes + const size1 = data1.byteLength + const blob1 = { + digest: digest1, + size: size1, + } + + // Get alice empty allocations + const allocationsAllice0 = await allocationsStorage.list(aliceSpaceDid) + assert.ok(allocationsAllice0.ok) + assert.deepEqual(allocationsAllice0.ok?.results, []) + assert.equal(allocationsAllice0.ok?.size, 0) + + // invoke `blob/add` with alice + const aliceBlobAdd0 = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: blob0, + }, + proofs: [aliceProof], + }) + const aliceInvocation = (await aliceBlobAdd0.delegate()).link() + + // Add alice allocations + const aliceAllocationInsert0 = await allocationsStorage.insert({ + space: aliceSpaceDid, + blob: blob0, + invocation: aliceInvocation, + }) + assert.ok(aliceAllocationInsert0.ok) + + // invoke `blob/add` with bob + const bobBlobAdd = BlobCapabilities.add.invoke({ + issuer: bob, + audience: context.id, + with: bobSpaceDid, + nb: { + blob: blob1, + }, + proofs: [bobProof], + }) + const invocation = (await bobBlobAdd.delegate()).link() + + // Add bob allocations + const bobAllocationInsert = await allocationsStorage.insert({ + space: bobSpaceDid, + blob: blob1, + invocation, + }) + assert.ok(bobAllocationInsert.ok) + + const allocationsAllice1 = await allocationsStorage.list(aliceSpaceDid) + assert.ok(allocationsAllice1.ok) + assert.equal(allocationsAllice1.ok?.size, 1) + assert.equal(allocationsAllice1.ok?.results.length, 1) + assert.ok( + equals( + blob0.digest, + allocationsAllice1.ok?.results[0].blob.digest || new Uint8Array() + ) + ) + + // Add bob's data on alice alloctions + const aliceBlobAdd01 = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: aliceSpaceDid, + nb: { + blob: blob1, + }, + proofs: [aliceProof], + }) + const aliceInvocation1 = (await aliceBlobAdd01.delegate()).link() + + // Add alice allocations + const aliceAllocationInsert1 = await allocationsStorage.insert({ + space: aliceSpaceDid, + blob: blob1, + invocation: aliceInvocation1, + }) + assert.ok(aliceAllocationInsert1.ok) + + const allocationsAllice2 = await allocationsStorage.list(aliceSpaceDid) + assert.ok(allocationsAllice2.ok) + assert.equal(allocationsAllice2.ok?.size, 2) + assert.equal(allocationsAllice2.ok?.results.length, 2) + assert.ok( + allocationsAllice2.ok?.results.find((res) => + equals(res.blob.digest, blob0.digest) + ) + ) + assert.ok( + allocationsAllice2.ok?.results.find((res) => + equals(res.blob.digest, blob1.digest) + ) + ) + }, +} diff --git a/packages/upload-api/test/storage/allocations-storage.js b/packages/upload-api/test/storage/allocations-storage.js index 115b5b3b8..ba3b06181 100644 --- a/packages/upload-api/test/storage/allocations-storage.js +++ b/packages/upload-api/test/storage/allocations-storage.js @@ -1,5 +1,6 @@ import * as Types from '../../src/types.js' import { equals } from 'uint8arrays/equals' +import { RecordKeyConflict, RecordNotFound } from '../../src/errors.js' /** * @implements {Types.AllocationsStorage} @@ -21,7 +22,7 @@ export class AllocationsStorage { ) ) { return { - error: { name: 'RecordKeyConflict', message: 'record key conflict' }, + error: new RecordKeyConflict(), } } this.items.unshift({ @@ -43,7 +44,7 @@ export class AllocationsStorage { (i) => i.space === space && equals(i.blob.digest, blobMultihash) ) if (!item) { - return { error: { name: 'RecordNotFound', message: 'record not found' } } + return { error: new RecordNotFound() } } return { ok: item } } diff --git a/packages/upload-api/test/storage/allocations-storage.spec.js b/packages/upload-api/test/storage/allocations-storage.spec.js new file mode 100644 index 000000000..817c610f1 --- /dev/null +++ b/packages/upload-api/test/storage/allocations-storage.spec.js @@ -0,0 +1,3 @@ +import * as AllocationsStorage from './allocations-storage-tests.js' +import { test } from '../test.js' +test({ 'in memory allocations storage': AllocationsStorage.test }) diff --git a/packages/upload-api/test/storage/blobs-storage-tests.js b/packages/upload-api/test/storage/blobs-storage-tests.js new file mode 100644 index 000000000..5936b6d47 --- /dev/null +++ b/packages/upload-api/test/storage/blobs-storage-tests.js @@ -0,0 +1,46 @@ +import * as API from '../../src/types.js' + +import { sha256 } from 'multiformats/hashes/sha2' + +/** + * @type {API.Tests} + */ +export const test = { + 'should create valid presigned URL for blobs that can be used to write': + async (assert, context) => { + const blobsStorage = context.blobsStorage + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash0 = await sha256.digest(data) + const digest = multihash0.bytes + const size = data.byteLength + const expiresIn = 60 * 60 * 24 // 1 day + const blob = { + digest: digest, + size: size, + } + const createUploadUrl = await blobsStorage.createUploadUrl( + blob.digest, + blob.size, + expiresIn + ) + if (!createUploadUrl.ok) { + throw new Error('should create presigned url') + } + + assert.ok(createUploadUrl.ok.headers['content-length']) + assert.ok(createUploadUrl.ok.headers['x-amz-checksum-sha256']) + + // Store the blob to the address + const goodPut = await fetch(createUploadUrl.ok.url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: createUploadUrl.ok.headers, + }) + assert.equal(goodPut.status, 200, await goodPut.text()) + + // check it exists + const hasBlob = await blobsStorage.has(blob.digest) + assert.ok(hasBlob.ok) + }, +} diff --git a/packages/upload-api/test/storage/blobs-storage.spec.js b/packages/upload-api/test/storage/blobs-storage.spec.js new file mode 100644 index 000000000..33682e9d4 --- /dev/null +++ b/packages/upload-api/test/storage/blobs-storage.spec.js @@ -0,0 +1,3 @@ +import * as BlobsStorage from './blobs-storage-tests.js' +import { test } from '../test.js' +test({ 'in memory blobs storage': BlobsStorage.test }) diff --git a/packages/upload-api/test/storage/receipts-storage-tests.js b/packages/upload-api/test/storage/receipts-storage-tests.js new file mode 100644 index 000000000..4c19a6e54 --- /dev/null +++ b/packages/upload-api/test/storage/receipts-storage-tests.js @@ -0,0 +1,108 @@ +import * as API from '../../src/types.js' + +import { sha256 } from 'multiformats/hashes/sha2' +import * as BlobCapabilities from '@web3-storage/capabilities/blob' + +import { RecordNotFoundErrorName } from '../../src/errors.js' +import { alice, registerSpace } from '../util.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'should be able to store receipts, even the same': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const receiptsStorage = context.receiptsStorage + const connection = context.connection + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + // Invoke `blob/add` + const receipt = await blobAdd.execute(connection) + if (!receipt.out.ok) { + throw new Error('invocation failed', { cause: receipt }) + } + + const putTask0 = await receiptsStorage.put(receipt) + assert.ok(putTask0.ok) + + // same put + const putTask1 = await receiptsStorage.put(receipt) + assert.ok(putTask1.ok) + }, + 'should be able to get stored receipts, or check if they exist': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const receiptsStorage = context.receiptsStorage + const connection = context.connection + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + // Invoke `blob/add` + const receipt = await blobAdd.execute(connection) + if (!receipt.out.ok) { + throw new Error('invocation failed', { cause: receipt }) + } + + // Get before put + const getTask0 = await receiptsStorage.get(receipt.ran.link()) + assert.ok(getTask0.error) + assert.equal(getTask0.error?.name, RecordNotFoundErrorName) + + // Has before put + const hasTask0 = await receiptsStorage.has(receipt.ran.link()) + assert.ok(!hasTask0.error) + assert.ok(!hasTask0.ok) + + // Put task + const putTask = await receiptsStorage.put(receipt) + assert.ok(putTask.ok) + + // Get after put + const getTask1 = await receiptsStorage.get(receipt.ran.link()) + assert.ok(getTask1.ok) + assert.ok(getTask1.ok?.ran.link().equals(receipt.ran.link())) + + // Has after put + const hasTask1 = await receiptsStorage.has(receipt.ran.link()) + assert.ok(!hasTask1.error) + assert.ok(hasTask1.ok) + }, +} diff --git a/packages/upload-api/test/storage/receipts-storage.spec.js b/packages/upload-api/test/storage/receipts-storage.spec.js new file mode 100644 index 000000000..b43c99d41 --- /dev/null +++ b/packages/upload-api/test/storage/receipts-storage.spec.js @@ -0,0 +1,3 @@ +import * as ReceiptsStorage from './receipts-storage-tests.js' +import { test } from '../test.js' +test({ 'in memory receipts storage': ReceiptsStorage.test }) diff --git a/packages/upload-api/test/storage/tasks-storage-tests.js b/packages/upload-api/test/storage/tasks-storage-tests.js new file mode 100644 index 000000000..5c51b817e --- /dev/null +++ b/packages/upload-api/test/storage/tasks-storage-tests.js @@ -0,0 +1,95 @@ +import * as API from '../../src/types.js' + +import { sha256 } from 'multiformats/hashes/sha2' +import * as BlobCapabilities from '@web3-storage/capabilities/blob' + +import { RecordNotFoundErrorName } from '../../src/errors.js' +import { alice, registerSpace } from '../util.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'should be able to store tasks, even the same': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const tasksStorage = context.tasksStorage + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const task = await blobAdd.delegate() + + const putTask0 = await tasksStorage.put(task) + assert.ok(putTask0.ok) + + // same put + const putTask1 = await tasksStorage.put(task) + assert.ok(putTask1.ok) + }, + 'should be able to get stored tasks, or check if they exist': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const tasksStorage = context.tasksStorage + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const task = await blobAdd.delegate() + + // Get before put + const getTask0 = await tasksStorage.get(task.link()) + assert.ok(getTask0.error) + assert.equal(getTask0.error?.name, RecordNotFoundErrorName) + + // Has before put + const hasTask0 = await tasksStorage.has(task.link()) + assert.ok(!hasTask0.error) + assert.ok(!hasTask0.ok) + + // Put task + const putTask = await tasksStorage.put(task) + assert.ok(putTask.ok) + + // Get after put + const getTask1 = await tasksStorage.get(task.link()) + assert.ok(getTask1.ok) + assert.ok(getTask1.ok?.link().equals(task.link())) + + // Has after put + const hasTask1 = await tasksStorage.has(task.link()) + assert.ok(!hasTask1.error) + assert.ok(hasTask1.ok) + }, +} diff --git a/packages/upload-api/test/storage/tasks-storage.js b/packages/upload-api/test/storage/tasks-storage.js index 7a704e6a2..7dfb00014 100644 --- a/packages/upload-api/test/storage/tasks-storage.js +++ b/packages/upload-api/test/storage/tasks-storage.js @@ -25,6 +25,14 @@ export class TasksStorage { async put(record) { this.items.set(record.cid.toString(), record) + // TODO: store implementation + // const archiveDelegationRes = await task.archive() + // if (archiveDelegationRes.error) { + // return { + // error: archiveDelegationRes.error + // } + // } + return Promise.resolve({ ok: {}, }) @@ -38,7 +46,7 @@ export class TasksStorage { const record = this.items.get(link.toString()) if (!record) { return { - error: new RecordNotFound('not found'), + error: new RecordNotFound(), } } return { diff --git a/packages/upload-api/test/storage/tasks-storage.spec.js b/packages/upload-api/test/storage/tasks-storage.spec.js new file mode 100644 index 000000000..afc9a397b --- /dev/null +++ b/packages/upload-api/test/storage/tasks-storage.spec.js @@ -0,0 +1,3 @@ +import * as TasksStorage from './tasks-storage-tests.js' +import { test } from '../test.js' +test({ 'in memory tasks storage': TasksStorage.test }) From 4cf99fa21e2d567d092ebdb55f4cf46b79297a1c Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 12 Apr 2024 09:56:00 +0200 Subject: [PATCH 26/27] chore: address last review comments --- packages/upload-api/src/blob/add.js | 10 ++++++---- packages/upload-api/src/blob/allocate.js | 3 +++ packages/upload-api/src/ucan/conclude.js | 2 ++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 17896ab67..0d0fdc533 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -159,7 +159,7 @@ async function allocate({ context, blob, space, cause }) { } } - // 3. if not already allocated (or expired) schedule `blob/allocate` + // 3. if not already allocated (or expired) execute `blob/allocate` if (!blobAllocateReceipt) { // Execute allocate invocation const allocateRes = await allocate.execute(context.getServiceConnection()) @@ -208,7 +208,7 @@ async function put({ context, blob, allocateTask }) { // could perform the invocation and issue receipt by deriving same // principal const blobProvider = await ed25519.derive( - blob.digest.slice(blob.digest.length - 32) + blob.digest.subarray(-32) ) const facts = [ { @@ -235,6 +235,8 @@ async function put({ context, blob, allocateTask }) { // 2. Get receipt for `http/put` if available const receiptGet = await context.receiptsStorage.get(task.link()) + // Storage get can fail with `RecordNotFound` or other unexpected errors. + // If 'RecordNotFound' we proceed, otherwise we fail with the received error. if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { return { error: receiptGet.error, @@ -306,7 +308,7 @@ async function accept({ context, blob, space, putTask, putReceipt }) { } } - // 3. Get receipt for `blob/accept` if available, otherwise schedule invocation + // 3. Get receipt for `blob/accept` if available, otherwise execute invocation let blobAcceptReceipt const receiptGet = await context.receiptsStorage.get(task.link()) if (receiptGet.error && receiptGet.error.name !== 'RecordNotFound') { @@ -317,7 +319,7 @@ async function accept({ context, blob, space, putTask, putReceipt }) { blobAcceptReceipt = receiptGet.ok } - // 4. if not already accepted schedule `blob/accept` + // 4. if not already accepted execute `blob/accept` if (!blobAcceptReceipt) { // Execute accept invocation const acceptRes = await accept.execute(context.getServiceConnection()) diff --git a/packages/upload-api/src/blob/allocate.js b/packages/upload-api/src/blob/allocate.js index 7abd151a7..a1f256040 100644 --- a/packages/upload-api/src/blob/allocate.js +++ b/packages/upload-api/src/blob/allocate.js @@ -51,12 +51,15 @@ export function blobAllocateProvider(context) { } // Check if blob already exists + // TODO: this may depend on the region we want to allocate and will need changes in the future. const hasBlobStore = await context.blobsStorage.has(blob.digest) if (hasBlobStore.error) { return hasBlobStore } // If blob is stored, we can just allocate it to the space with the allocated size + // TODO: this code path MAY lead to await failures - awaited http/put and blob/accept tasks + // are supposed to fail if path does not exists. if (hasBlobStore.ok) { return { ok: { size }, diff --git a/packages/upload-api/src/ucan/conclude.js b/packages/upload-api/src/ucan/conclude.js index dcc49cb2d..564811efe 100644 --- a/packages/upload-api/src/ucan/conclude.js +++ b/packages/upload-api/src/ucan/conclude.js @@ -82,6 +82,8 @@ export const ucanConcludeProvider = ({ 'ucan/await': ['.out.ok', ranInvocation.link()], }, }, + // Expiry is set to `Infinity` so that CID will come out the same + // as returned in effect on `blob/add` expiration: Infinity, }) .delegate() From 15d732c3aa8442a0ee7c1cef247d028384bb5823 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 12 Apr 2024 10:19:10 +0200 Subject: [PATCH 27/27] fix: remove unused dep --- pnpm-lock.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d4f4b5479..d1ac87d4b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -431,9 +431,6 @@ importers: '@types/mocha': specifier: ^10.0.1 version: 10.0.4 - '@types/sinon': - specifier: ^17.0.3 - version: 17.0.3 '@ucanto/core': specifier: ^10.0.1 version: 10.0.1 @@ -4103,12 +4100,6 @@ packages: '@types/sinonjs__fake-timers': 8.1.5 dev: true - /@types/sinon@17.0.3: - resolution: {integrity: sha512-j3uovdn8ewky9kRBG19bOwaZbexJu/XjtkHyjvUgt4xfPFz18dcORIMqnYh66Fx3Powhcr85NT5+er3+oViapw==} - dependencies: - '@types/sinonjs__fake-timers': 8.1.5 - dev: true - /@types/sinonjs__fake-timers@8.1.5: resolution: {integrity: sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==} dev: true