From bb21369e477c58f43a876699804989a9176d4c66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=C3=A3o=20Sousa=20Andrade?=
 <659718+joaosa@users.noreply.github.com>
Date: Mon, 29 Apr 2024 16:53:19 +0100
Subject: [PATCH] feat: add the EIPFS infra for the blob protocol (#361)

- adds the EIPFS multihashes queue;
- adds the EIPFS blocks CAR positions table.
---
 .env.tpl                   |  2 ++
 README.md                  |  8 ++++++++
 stacks/config.js           |  2 ++
 stacks/upload-api-stack.js | 33 +++++++++++++++++++++++++++++++--
 4 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/.env.tpl b/.env.tpl
index f108d9ee..7ca6eef1 100644
--- a/.env.tpl
+++ b/.env.tpl
@@ -13,6 +13,8 @@
 
 EIPFS_INDEXER_SQS_ARN = 'arn:aws:sqs:us-west-2:505595374361:staging-ep-indexer-topic'
 EIPFS_INDEXER_SQS_URL = 'https://sqs.us-west-2.amazonaws.com/505595374361/staging-ep-indexer-topic'
+EIPFS_MULTIHASHES_SQS_ARN = 'arn:aws:sqs:us-west-2:505595374361:staging-ep-multihashes-topic'
+EIPFS_BLOCKS_CAR_POSITION_TABLE_ARN = 'arn:aws:dynamodb:us-west-2:505595374361:table/staging-ep-v1-blocks-cars-position'
 
 PROVIDERS = ''
 UPLOAD_API_DID = ''
diff --git a/README.md b/README.md
index c6f3862c..b11d0d9f 100644
--- a/README.md
+++ b/README.md
@@ -199,6 +199,14 @@ AWS ARN for Elastic IPFS SQS indexer used to request Elastic IPFS to index given
 
 AWS URL for Elastic IPFS SQS indexer used to request Elastic IPFS to index given CAR files.
 
+#### `EIPFS_MULTIHASHES_SQS_ARN`
+
+AWS ARN for Elastic IPFS SQS multihashes used to enqueue multihashes for indexed CAR files.
+
+#### `EIPFS_BLOCKS_CAR_POSITION_TABLE_ARN`
+
+AWS ARN for Elastic IPFS DynamoDB table used to store blocks and positions for indexed CAR files.
+
 #### `POSTMARK_TOKEN`
 
 Postmark API token, which is used by the email verification system to send emails.
diff --git a/stacks/config.js b/stacks/config.js
index 5694c2f2..3596cd15 100644
--- a/stacks/config.js
+++ b/stacks/config.js
@@ -176,6 +176,8 @@ export function getEnv() {
     AGGREGATOR_URL: mustGetEnv('AGGREGATOR_URL'),
     CONTENT_CLAIMS_DID: mustGetEnv('CONTENT_CLAIMS_DID'),
     CONTENT_CLAIMS_URL: mustGetEnv('CONTENT_CLAIMS_URL'),
+    EIPFS_MULTIHASHES_SQS_ARN: mustGetEnv('EIPFS_MULTIHASHES_SQS_ARN'),
+    EIPFS_BLOCKS_CAR_POSITION_TABLE_ARN: mustGetEnv('EIPFS_BLOCKS_CAR_POSITION_TABLE_ARN'),
     // Not required
     STOREFRONT_PROOF: process.env.STOREFRONT_PROOF ?? '',
     CONTENT_CLAIMS_PROOF: process.env.CONTENT_CLAIMS_PROOF ?? '',
diff --git a/stacks/upload-api-stack.js b/stacks/upload-api-stack.js
index 63d420c3..1368b2b1 100644
--- a/stacks/upload-api-stack.js
+++ b/stacks/upload-api-stack.js
@@ -3,8 +3,12 @@ import {
   Config,
   Function,
   Queue,
+  Table,
   use
 } from 'sst/constructs'
+import * as sqs from 'aws-cdk-lib/aws-sqs'
+import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'
+
 import { StartingPosition } from 'aws-cdk-lib/aws-lambda'
 import { UploadDbStack } from './upload-db-stack.js'
 import { BillingDbStack } from './billing-db-stack.js'
@@ -18,7 +22,7 @@ import { getCustomDomain, getApiPackageJson, getGitInfo, setupSentry, getEnv, ge
  * @param {import('sst/constructs').StackContext} properties
  */
 export function UploadApiStack({ stack, app }) {
-  const { AGGREGATOR_DID } = getEnv()
+  const { AGGREGATOR_DID, EIPFS_MULTIHASHES_SQS_ARN, EIPFS_BLOCKS_CAR_POSITION_TABLE_ARN } = getEnv()
 
   // Setup app monitoring with Sentry
   setupSentry(app, stack)
@@ -30,6 +34,28 @@ export function UploadApiStack({ stack, app }) {
   const { customerTable, spaceDiffTable, spaceSnapshotTable, stripeSecretKey } = use(BillingDbStack)
   const { pieceOfferQueue, filecoinSubmitQueue } = use(FilecoinStack)
 
+  // Blob protocol
+  // Elastic IPFS event for multihashes
+  const multihashesQueue = new Queue(stack, 'multihashes-topic-queue', {
+    cdk: {
+      queue: sqs.Queue.fromQueueArn(
+        stack,
+        'multihashes-topic',
+        EIPFS_MULTIHASHES_SQS_ARN
+      ),
+    },
+  })
+
+  const blocksCarPositionTable = new Table(stack, 'blocks-car-position-table', {
+    cdk: {
+      table: dynamodb.Table.fromTableArn(
+        stack,
+        'blocks-car-position',
+        EIPFS_BLOCKS_CAR_POSITION_TABLE_ARN
+      ),
+    },
+  })
+
   // Setup API
   const customDomain = getCustomDomain(stack.stage, process.env.HOSTED_ZONE)
   const pkg = getApiPackageJson()
@@ -62,7 +88,8 @@ export function UploadApiStack({ stack, app }) {
           workflowBucket,
           ucanStream,
           pieceOfferQueue,
-          filecoinSubmitQueue
+          filecoinSubmitQueue,
+          multihashesQueue,
         ],
         environment: {
           DID: process.env.UPLOAD_API_DID ?? '',
@@ -89,6 +116,8 @@ export function UploadApiStack({ stack, app }) {
           PIECE_TABLE_NAME: pieceTable.tableName,
           PIECE_OFFER_QUEUE_URL: pieceOfferQueue.queueUrl,
           FILECOIN_SUBMIT_QUEUE_URL: filecoinSubmitQueue.queueUrl,
+          MULTIHASHES_QUEUE_URL: multihashesQueue.queueUrl,
+          BLOCKS_CAR_POSITION_TABLE_NAME: blocksCarPositionTable.tableName,
           NAME: pkg.name,
           VERSION: pkg.version,
           COMMIT: git.commmit,