Merge branch 'main' into fix/use-multihashdigest-type1

.github/workflows/manual.yml

@@ -69,10 +69,3 @@ jobs:
       - run: pnpm -r --filter @web3-storage/upload-client publish --tag next --access public
         env:
           NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
-  deploy-docs:
-    permissions:
-      contents: read
-      pages: write
-      id-token: write
-    if: github.event.inputs.package == 'docs'
-    uses: './.github/workflows/reusable-deploy-docs.yml'

.github/workflows/release.yml

@@ -51,20 +51,3 @@ jobs:
       - run: pnpm -r publish --access=public
         env:
           NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}
-  docs:
-    permissions:
-      contents: read
-      pages: write
-      id-token: write
-    needs: release
-    if: |
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/access-client') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/blob-index') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/capabilities') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/did-mailto') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/upload-client') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/upload-api') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/filecoin-client') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/filecoin-api') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/w3up-client')
-    uses: './.github/workflows/reusable-deploy-docs.yml'

packages/capabilities/src/index/index.js

@@ -39,7 +39,7 @@ export const add = capability({
   with: SpaceDID,
   nb: Schema.struct({
     /** Content Archive (CAR) containing the `Index`. */
-    index: Schema.link({ code: CAR.code }),
+    index: Schema.link({ code: CAR.code, version: 1 }),
   }),
   derives: (claimed, delegated) =>
     and(equalWith(claimed, delegated)) ||

packages/upload-api/package.json

@@ -201,11 +201,10 @@
     "@web3-storage/access": "workspace:^",
     "@web3-storage/blob-index": "workspace:^",
     "@web3-storage/capabilities": "workspace:^",
-    "@web3-storage/content-claims": "^5.0.0",
+    "@web3-storage/content-claims": "^5.1.0",
     "@web3-storage/did-mailto": "workspace:^",
     "@web3-storage/filecoin-api": "workspace:^",
     "multiformats": "^12.1.2",
-    "p-retry": "^5.1.2",
     "uint8arrays": "^5.0.3"
   },
   "devDependencies": {

packages/upload-api/src/index/add.js

@@ -2,6 +2,7 @@ import * as Server from '@ucanto/server'
 import { ok, error } from '@ucanto/server'
 import * as Index from '@web3-storage/capabilities/index'
 import { ShardedDAGIndex } from '@web3-storage/blob-index'
+import { Assert } from '@web3-storage/content-claims/capability'
 import { concat } from 'uint8arrays'
 import * as API from '../types.js'
 
@@ -61,13 +62,21 @@ const add = async ({ capability }, context) => {
     shardDigests.map((s) => assertAllocated(context, space, s, 'ShardNotFound'))
   )
   for (const res of shardAllocRes) {
-    if (!res.ok) return res
+    if (res.error) return res
   }
 
   // TODO: randomly validate slices in the index correspond to slices in the blob
 
-  // publish the index data to IPNI
-  return context.ipniService.publish(idxRes.ok)
+  const publishRes = await Promise.all([
+    // publish the index data to IPNI
+    context.ipniService.publish(idxRes.ok),
+    // publish a content claim for the index
+    publishIndexClaim(context, { content: idxRes.ok.content, index: idxLink }),
+  ])
+  for (const res of publishRes) {
+    if (res.error) return res
+  }
+  return ok({})
 }
 
 /**
@@ -87,3 +96,23 @@ const assertAllocated = async (context, space, digest, errorName) => {
     )
   return ok({})
 }
+
+/**
+ * @param {API.ClaimsClientContext} ctx
+ * @param {{ content: API.UnknownLink, index: API.CARLink }} params
+ */
+const publishIndexClaim = async (ctx, { content, index }) => {
+  const { invocationConfig, connection } = ctx.claimsService
+  const { issuer, audience, with: resource, proofs } = invocationConfig
+  const res = await Assert.index
+    .invoke({
+      issuer,
+      audience,
+      with: resource,
+      nb: { content, index },
+      expiration: Infinity,
+      proofs,
+    })
+    .execute(connection)
+  return res.out
+}

packages/upload-api/src/types.ts

@@ -204,13 +204,21 @@ import { StorageGetError } from './types/storage.js'
 import { AllocationsStorage, BlobsStorage, BlobAddInput } from './types/blob.js'
 export type { AllocationsStorage, BlobsStorage, BlobAddInput }
 import { IPNIService, IndexServiceContext } from './types/index.js'
+import { ClaimsClientConfig } from './types/content-claims.js'
+import { Claim } from '@web3-storage/content-claims/client/api'
 export type {
   IndexServiceContext,
   IPNIService,
   BlobRetriever,
   BlobNotFound,
   ShardedDAGIndex,
 } from './types/index.js'
+export type {
+  ClaimsInvocationConfig,
+  ClaimsClientConfig,
+  ClaimsClientContext,
+  Service as ClaimsService,
+} from './types/content-claims.js'
 
 export interface Service extends StorefrontService, W3sService {
   store: {
@@ -392,7 +400,6 @@ export type UploadServiceContext = ConsumerServiceContext &
   ConcludeServiceContext & {
     signer: EdSigner.Signer
     uploadTable: UploadTable
-    dudewhereBucket: DudewhereBucket
   }
 
 export interface AccessClaimContext {
@@ -591,6 +598,18 @@ export interface UcantoServerTestContext
   ipniService: IPNIService & {
     query(digest: MultihashDigest): Promise<Result<Unit, RecordNotFound>>
   }
+
+  carStoreBucket: CarStoreBucket & Deactivator
+  blobsStorage: BlobsStorage & Deactivator
+  claimsService: ClaimsClientConfig & ClaimReader & Deactivator
+}
+
+export interface ClaimReader {
+  read(digest: MultihashDigest): Promise<Result<Claim[], Failure>>
+}
+
+export interface Deactivator {
+  deactivate: () => Promise<void>
 }
 
 export interface StoreTestContext {}
@@ -628,10 +647,6 @@ export interface CarStoreBucketService {
   use(options?: CarStoreBucketOptions): Promise<CarStoreBucket>
 }
 
-export interface DudewhereBucket {
-  put: (dataCid: string, carCid: string) => Promise<void>
-}
-
 /**
  * Indicates the requested record was not present in the table.
  */

packages/upload-api/src/types/content-claims.ts

@@ -0,0 +1,31 @@
+import {
+  ConnectionView,
+  DID,
+  Principal,
+  Proof,
+  Signer,
+} from '@ucanto/interface'
+import { Service } from '@web3-storage/content-claims/server/service/api'
+
+export type { ConnectionView, DID, Principal, Proof, Signer }
+export type { Service }
+
+export interface ClaimsInvocationConfig {
+  /** Signing authority issuing the UCAN invocation(s). */
+  issuer: Signer
+  /** The principal delegated to in the current UCAN. */
+  audience: Principal
+  /** The resource the invocation applies to. */
+  with: DID
+  /** Proof(s) the issuer has the capability to perform the action. */
+  proofs?: Proof[]
+}
+
+export interface ClaimsClientConfig {
+  invocationConfig: ClaimsInvocationConfig
+  connection: ConnectionView<Service>
+}
+
+export interface ClaimsClientContext {
+  claimsService: ClaimsClientConfig
+}

packages/upload-api/src/types/index.ts

@@ -2,8 +2,9 @@ import { MultihashDigest } from 'multiformats'
 import { Failure, Result, Unit } from '@ucanto/interface'
 import { ShardedDAGIndex } from '@web3-storage/blob-index/types'
 import { AllocationsStorage } from './blob.js'
+import { ClaimsClientContext } from './content-claims.js'
 
-export type { ShardedDAGIndex }
+export type { ShardedDAGIndex, ClaimsClientContext }
 
 /**
  * Service that allows publishing a set of multihashes to IPNI for a
@@ -26,7 +27,7 @@ export interface BlobRetriever {
   ): Promise<Result<ReadableStream<Uint8Array>, BlobNotFound>>
 }
 
-export interface IndexServiceContext {
+export interface IndexServiceContext extends ClaimsClientContext {
   allocationsStorage: AllocationsStorage
   blobRetriever: BlobRetriever
   ipniService: IPNIService

packages/upload-api/src/upload/add.js

@@ -1,4 +1,3 @@
-import pRetry from 'p-retry'
 import * as Server from '@ucanto/server'
 import * as Upload from '@web3-storage/capabilities/upload'
 import * as API from '../types.js'
@@ -10,7 +9,7 @@ import { allocate } from '../space-allocate.js'
  */
 export function uploadAddProvider(context) {
   return Server.provide(Upload.add, async ({ capability, invocation }) => {
-    const { uploadTable, dudewhereBucket } = context
+    const { uploadTable } = context
     const { root, shards } = capability.nb
     const space = /** @type {import('@ucanto/interface').DIDKey} */ (
       Server.DID.parse(capability.with).did()
@@ -28,39 +27,12 @@ export function uploadAddProvider(context) {
       return allocated
     }
 
-    const [res] = await Promise.all([
-      // Store in Database
-      uploadTable.upsert({
-        space,
-        root,
-        shards,
-        issuer,
-        invocation: invocation.cid,
-      }),
-      writeDataCidToCarCidsMapping(dudewhereBucket, root, shards),
-    ])
-
-    return res
-  })
-}
-
-/**
- * Writes to a "bucket DB" the mapping from a data CID to the car CIDs it is composed of.
- * Retries up to 3 times, in case of failures.
- *
- * @param {import("../types.js").DudewhereBucket} dudewhereStore
- * @param {Server.API.Link<unknown, number, number, 0 | 1>} root
- * @param {Server.API.Link<unknown, 514, number, 1>[] | undefined} shards
- */
-async function writeDataCidToCarCidsMapping(dudewhereStore, root, shards) {
-  const dataCid = root.toString()
-  const carCids =
-    shards?.map((/** @type {{ toString: () => any; }} */ s) => s.toString()) ||
-    []
-
-  return Promise.all(
-    carCids.map(async (/** @type {string} */ carCid) => {
-      await pRetry(() => dudewhereStore.put(dataCid, carCid), { retries: 3 })
+    return uploadTable.upsert({
+      space,
+      root,
+      shards,
+      issuer,
+      invocation: invocation.cid,
     })
-  )
+  })
 }