feat: update upload-api dependency (#390)

Updates `upload-api` dependency, bringing in a breaking change to blob
storage (digest parameter type changed from `Uint8Array` to
`MultihashDigest`), a change to `index/add` handler that publishes an
index claim to content claims (needed implementation of content claims
service for tests) and finally a change that drops writing to DUDEWHERE
R2 bucket since it is no longer used in the reads pipeline.

see storacha/w3up#1501

.env.tpl

@@ -27,10 +27,8 @@ DEAL_TRACKER_URL = ''
 POSTMARK_TOKEN = ''
 R2_ACCESS_KEY_ID = ''
 R2_CARPARK_BUCKET_NAME = ''
-R2_DUDEWHERE_BUCKET_NAME = ''
 R2_ENDPOINT = ''
 R2_REGION = ''
-R2_SATNAV_BUCKET_NAME = ''
 R2_SECRET_ACCESS_KEY = ''
 R2_UCAN_BUCKET_NAME = ''
 R2_DELEGATION_BUCKET_NAME = ''

README.md

@@ -170,14 +170,6 @@ Endpoint for S3 like cloud object storage to replicate content into.
 
 Bucket name to replicate written CAR files.
 
-#### `R2_SATNAV_BUCKET_NAME`
-
-Bucket name to replicate written .idx files.
-
-#### `R2_DUDEWHERE_BUCKET_NAME`
-
-Bucket name to replicate root CID to car CIDs mapping.
-
 #### `R2_DELEGATION_BUCKET_NAME`
 
 Bucket name where delegations are stored.

carpark/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.0",
   "type": "module",
   "scripts": {
-    "test": "ava --verbose --timeout=60s **/*.test.js"
+    "test": "ava --verbose --timeout=60s --fail-fast **/*.test.js"
   },
   "dependencies": {
     "@aws-sdk/client-eventbridge": "^3.515.0",

filecoin/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.0",
   "type": "module",
   "scripts": {
-    "test": "ava --serial --no-worker-threads --verbose --timeout=60s **/*.test.js"
+    "test": "ava --serial --no-worker-threads --verbose --timeout=60s --fail-fast **/*.test.js"
   },
   "dependencies": {
     "@aws-sdk/client-dynamodb": "^3.515.0",

package.json

@@ -13,19 +13,19 @@
     "lint": "tsc && eslint '**/*.js'",
     "clean": "rm -rf dist node_modules package-lock.json ./*/{.cache,dist,node_modules}",
     "test": "npm test -w billing -w upload-api -w carpark -w replicator -w roundabout -w filecoin",
-    "test-integration": "ava --verbose --serial --timeout=660s test/*.test.js",
+    "test-integration": "ava --verbose --serial --timeout=660s --fail-fast test/*.test.js",
     "fetch-metrics-for-space": "npm run fetch-metrics-for-space -w tools",
     "follow-filecoin-receipt-chain": "npm run follow-filecoin-receipt-chain -w tools",
     "d1-dynamo-migration": "npm run d1-dynamo-migration -w tools"
   },
   "devDependencies": {
     "@chainsafe/libp2p-noise": "^15.0.0",
+    "@helia/unixfs": "^3.0.6",
     "@ipld/car": "^5.2.6",
     "@ipld/dag-ucan": "^3.0.1",
-    "@helia/unixfs": "^3.0.6",
     "@libp2p/identify": "^2.0.1",
-    "@libp2p/websockets": "^8.0.23",
     "@libp2p/mplex": "^10.0.23",
+    "@libp2p/websockets": "^8.0.23",
     "@multiformats/multiaddr": "^12.2.1",
     "@tsconfig/node16": "^1.0.3",
     "@types/git-rev-sync": "^2.0.0",
@@ -39,8 +39,8 @@
     "@web-std/fetch": "^4.1.0",
     "@web3-storage/access": "^20.0.0",
     "@web3-storage/blob-index": "^1.0.2",
-    "@web3-storage/content-claims": "^5.1.0",
     "@web3-storage/capabilities": "^17.2.0",
+    "@web3-storage/content-claims": "^5.1.0",
     "@web3-storage/data-segment": "5.1.0",
     "@web3-storage/filecoin-client": "^3.3.3",
     "@web3-storage/upload-client": "^16.1.0",
@@ -50,8 +50,8 @@
     "constructs": "10.3.0",
     "dotenv": "^16.0.3",
     "git-rev-sync": "^3.0.2",
-    "helia": "^4.2.1",
     "hd-scripts": "^3.0.2",
+    "helia": "^4.2.1",
     "libp2p": "1.5.2",
     "lint-staged": "^13.0.3",
     "mailslurp-client": "^15.17.2",
@@ -63,8 +63,8 @@
   "dependencies": {
     "@ipld/dag-json": "^10.1.5",
     "@web-std/stream": "^1.0.3",
+    "@web3-storage/upload-api": "^18.0.0",
     "aws-cdk-lib": "2.142.1",
-    "@web3-storage/upload-api": "^17.0.0",
     "sst": "^2.40.3"
   },
   "simple-git-hooks": {

replicator/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.0",
   "type": "module",
   "scripts": {
-    "test": "ava --verbose --timeout=60s **/*.test.js"
+    "test": "ava --verbose --timeout=60s --fail-fast **/*.test.js"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.515.0",

roundabout/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.0",
   "type": "module",
   "scripts": {
-    "test": "ava --verbose --timeout=60s **/*.test.js"
+    "test": "ava --verbose --timeout=60s --fail-fast **/*.test.js"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.515.0",

stacks/upload-api-stack.js

@@ -22,14 +22,21 @@ import { getCustomDomain, getApiPackageJson, getGitInfo, setupSentry, getEnv, ge
  * @param {import('sst/constructs').StackContext} properties
  */
 export function UploadApiStack({ stack, app }) {
-  const { AGGREGATOR_DID, EIPFS_MULTIHASHES_SQS_ARN, EIPFS_BLOCKS_CAR_POSITION_TABLE_ARN } = getEnv()
+  const {
+    AGGREGATOR_DID,
+    CONTENT_CLAIMS_DID,
+    CONTENT_CLAIMS_URL,
+    CONTENT_CLAIMS_PROOF,
+    EIPFS_MULTIHASHES_SQS_ARN,
+    EIPFS_BLOCKS_CAR_POSITION_TABLE_ARN
+  } = getEnv()
 
   // Setup app monitoring with Sentry
   setupSentry(app, stack)
 
   // Get references to constructs created in other stacks
   const { carparkBucket } = use(CarparkStack)
-  const { allocationTable, storeTable, uploadTable, delegationBucket, delegationTable, revocationTable, adminMetricsTable, spaceMetricsTable, consumerTable, subscriptionTable, rateLimitTable, pieceTable, privateKey } = use(UploadDbStack)
+  const { allocationTable, storeTable, uploadTable, delegationBucket, delegationTable, revocationTable, adminMetricsTable, spaceMetricsTable, consumerTable, subscriptionTable, rateLimitTable, pieceTable, privateKey, contentClaimsPrivateKey } = use(UploadDbStack)
   const { invocationBucket, taskBucket, workflowBucket, ucanStream } = use(UcanInvocationStack)
   const { customerTable, spaceDiffTable, spaceSnapshotTable, stripeSecretKey } = use(BillingDbStack)
   const { pieceOfferQueue, filecoinSubmitQueue } = use(FilecoinStack)
@@ -134,7 +141,6 @@ export function UploadApiStack({ stack, app }) {
           R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY ?? '',
           R2_REGION: process.env.R2_REGION ?? '',
           R2_CARPARK_BUCKET_NAME: process.env.R2_CARPARK_BUCKET_NAME ?? '',
-          R2_DUDEWHERE_BUCKET_NAME: process.env.R2_DUDEWHERE_BUCKET_NAME ?? '',
           R2_DELEGATION_BUCKET_NAME: process.env.R2_DELEGATION_BUCKET_NAME ?? '',
           R2_ENDPOINT: process.env.R2_ENDPOINT ?? '',
           REQUIRE_PAYMENT_PLAN: process.env.REQUIRE_PAYMENT_PLAN ?? '',
@@ -143,12 +149,16 @@ export function UploadApiStack({ stack, app }) {
           STRIPE_PUBLISHABLE_KEY: process.env.STRIPE_PUBLISHABLE_KEY ?? '',
           DEAL_TRACKER_DID: process.env.DEAL_TRACKER_DID ?? '',
           DEAL_TRACKER_URL: process.env.DEAL_TRACKER_URL ?? '',
-          INDEXER_REGION: indexerRegion
+          INDEXER_REGION: indexerRegion,
+          CONTENT_CLAIMS_DID,
+          CONTENT_CLAIMS_URL,
+          CONTENT_CLAIMS_PROOF
         },
         bind: [
           privateKey,
           ucanInvocationPostbasicAuth,
-          stripeSecretKey
+          stripeSecretKey,
+          contentClaimsPrivateKey
         ]
       }
     },

test/store.test.js

@@ -121,15 +121,6 @@ test('store protocol integration flow', async t => {
   t.is(carparkRequest.$metadata.httpStatusCode, 200)
   const carSize = carparkRequest.ContentLength
 
-  // Check dudewhere
-  const dudewhereRequest = await r2Client.send(
-    new HeadObjectCommand({
-      Bucket: process.env.R2_DUDEWHERE_BUCKET_NAME || '',
-      Key: `${root.toString()}/${shards[0].toString()}`
-    })
-  )
-  t.is(dudewhereRequest.$metadata.httpStatusCode, 200)
-
   // List space files
   let uploadFound, cursor
   do {
@@ -172,13 +163,22 @@ test('store protocol integration flow', async t => {
 
   // Verify w3link can resolve uploaded file
   console.log('Checking w3link can fetch root', root.toString())
-  const w3linkResponse = await fetch(
-    `https://${root}.ipfs-staging.w3s.link`,
-    {
-      method: 'HEAD'
+  const gatewayURL = `https://${root}.ipfs-staging.w3s.link`
+  const gatewayRetries = 5
+  for (let i = 0; i < gatewayRetries; i++) {
+    const controller = new AbortController()
+    const timeoutID = setTimeout(() => controller.abort(), 5000)
+    try {
+      const res = await fetch(gatewayURL, { method: 'HEAD', signal: controller.signal })
+      if (res.status === 200) break
+    } catch (err) {
+      console.error(`failed gateway fetch: ${root} (attempt ${i + 1})`, err)
+      if (i === gatewayRetries - 1) throw err
+    } finally {
+      clearTimeout(timeoutID)
     }
-  )
-  t.is(w3linkResponse.status, 200)
+    await new Promise(resolve => setTimeout(resolve, 1000))
+  }
 
   // Read from Roundabout returns 200
   console.log('Checking Roundabout can fetch CAR:', shards[0].toString())

upload-api/buckets/car-store.js

@@ -109,8 +109,9 @@ export function useCarStore(s3, bucketName) {
  * store#createUploadUrl is from first store.
  * store#has will check stores in order until 0-1 `true` are found.
  * 
- * @param  {import('@web3-storage/upload-api').CarStoreBucket} carStore 
- * @param  {Array<import('@web3-storage/upload-api').CarStoreBucket>} moreCarStores 
+ * @template {import('@web3-storage/upload-api').CarStoreBucket} T
+ * @param {T} carStore
+ * @param {T[]} moreCarStores
  */
 export function composeCarStoresWithOrderedHas(carStore, ...moreCarStores) {
   return {