Emulate Azure blob storage and AWS S3 (#2461)

* Add scripts to run azure or s3 locally

* Update local-dev script to choose storage based on SOURCE from .env

* [pre-commit.ci] auto fixes from pre-commit hooks

* Replace localstack with localstack-persist

* Update .env comments

---------

Co-authored-by: angelathe <angela.the@skylight.digital>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

containers/ecr-viewer/.env

@@ -1,8 +1,14 @@
 DATABASE_URL=postgres://postgres:pw@localhost:5432/ecr_viewer_db
+AWS_CUSTOM_ENDPOINT="http://localhost:4566"
 AWS_REGION=us-east-1
 ECR_BUCKET_NAME=ecr-viewer-files
-AZURE_STORAGE_CONNECTION_STRING=idk
-AZURE_CONTAINER_NAME=idk
+# These are the default credentials for localstack
+AWS_ACCESS_KEY_ID=123
+AWS_SECRET_ACCESS_KEY=xyz
+# NOT A REAL CONNECTION STRING! This is the default connection string for Azurite
+# https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio%2Cblob-storage#well-known-storage-account-and-key
+AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://0.0.0.0:10000/devstoreaccount1;"
+AZURE_CONTAINER_NAME=ecr-viewer-files
 SOURCE=postgres
 NEXT_TELEMETRY_DISABLED=1
 NEXT_PUBLIC_NON_INTEGRATED_VIEWER=true

containers/ecr-viewer/.gitignore

@@ -1,3 +1,6 @@
 node_modules
 .next/
-.env.local
+.env.local
+
+# localstack data
+volume/

containers/ecr-viewer/check-source-env.mjs

@@ -0,0 +1,6 @@
+import dotenv from "dotenv";
+
+dotenv.config();
+dotenv.config({ path: ".env.local", override: true });
+
+console.log(`local-dev-${process.env.SOURCE}`);

containers/ecr-viewer/compose.storage.yml

@@ -0,0 +1,17 @@
+services:
+  # Azure Blob Storage emulator
+  azure-storage:
+    image: mcr.microsoft.com/azure-storage/azurite
+    hostname: azurite
+    restart: always
+    command: "azurite --blobHost 0.0.0.0 --blobPort 10000 -l data"
+    ports:
+      - "10000:10000"
+
+  # AWS S3 Storage emulator
+  aws-storage:
+    image: gresau/localstack-persist:3
+    ports:
+      - "4566:4566"
+    volumes:
+      - "./volume:/persisted-data"

containers/ecr-viewer/environment.d.ts

@@ -3,6 +3,11 @@ namespace NodeJS {
   interface ProcessEnv {
     APP_ENV: "test" | "middleware" | "prod";
     AWS_REGION: string;
+    AWS_CUSTOM_ENDPOINT: string;
+    AWS_ACCESS_KEY_ID: string;
+    AWS_SECRET_ACCESS_KEY: string;
+    AZURE_STORAGE_CONNECTION_STRING: string;
+    AZURE_CONTAINER_NAME: string;
     DATABASE_SCHEMA: "core" | "extended";
     DATABASE_TYPE: string;
     DATABASE_URL: string;

containers/ecr-viewer/package.json

@@ -5,8 +5,13 @@
   "private": true,
   "scripts": {
     "dev": "next dev",
-    "local-dev": "npm run docker-compose-with-logs && npm run dev",
+    "local-dev": "npm run $(node ./check-source-env.mjs)",
+    "local-dev-postgres": "npm run docker-compose-with-logs && npm run dev",
+    "local-dev-azure": "npm run docker-compose-azure && npm run dev",
+    "local-dev-s3": "npm run docker-compose-aws && npm run dev",
     "docker-compose-with-logs": "docker compose up db -d && docker compose logs",
+    "docker-compose-azure": "docker compose -f docker-compose.yml -f compose.storage.yml up db azure-storage -d && docker compose logs",
+    "docker-compose-aws": "docker compose -f docker-compose.yml -f compose.storage.yml up aws-storage db -d && docker compose logs",
     "setup-local-env": "./setup-env.sh",
     "build": "next build",
     "start": "next start",
@@ -75,6 +80,7 @@
     "autoprefixer": "^10.4.19",
     "aws-sdk-client-mock": "^3.1.0",
     "cypress": "^13.6.6",
+    "dotenv": "^16.4.5",
     "eslint": "^8.56.0",
     "eslint-config-next": "14.0.3",
     "eslint-config-prettier": "^9.1.0",
@@ -90,4 +96,4 @@
     "sass": "^1.69.5",
     "typescript": "^5.5.2"
   }
-}
+}

containers/ecr-viewer/src/app/api/fhir-data/fhir-data-service.ts

@@ -9,7 +9,11 @@ import {
 import { loadYamlConfig, streamToJson } from "../utils";
 import { database } from "@/app/api/fhir-data/db";
 
-const s3Client = new S3Client({ region: process.env.AWS_REGION });
+const s3Client = new S3Client({
+  region: process.env.AWS_REGION,
+  endpoint: process.env.AWS_CUSTOM_ENDPOINT,
+  forcePathStyle: process.env.AWS_CUSTOM_ENDPOINT !== undefined,
+});
 
 /**
  * Retrieves FHIR data from PostgreSQL database based on eCR ID.

containers/ecr-viewer/src/app/api/save-fhir-data/save-fhir-data-service.ts

@@ -8,7 +8,11 @@ import {
 } from "@aws-sdk/client-s3";
 import { Bundle } from "fhir/r4";
 
-const s3Client = new S3Client({ region: process.env.AWS_REGION });
+const s3Client = new S3Client({
+  region: process.env.AWS_REGION,
+  endpoint: process.env.AWS_CUSTOM_ENDPOINT,
+  forcePathStyle: process.env.AWS_CUSTOM_ENDPOINT !== undefined,
+});
 
 /**
  * Saves a FHIR bundle to a postgres database.