Add all tasks to compose, update app to not run a worker, pin numpy t…

…o v1

docker-compose.yml

@@ -100,3 +100,78 @@ services:
         condition: service_healthy
       redis:
         condition: service_healthy
+  video:
+    build: .
+    platform: linux/amd64
+    volumes:
+      - "./:/app"
+    env_file:
+      - ./.env_file
+    environment: 
+      ROLE: worker
+      MODEL_NAME: video.Model
+    depends_on:
+      elasticmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+  mean_tokens:
+    build: .
+    platform: linux/amd64
+    volumes:
+      - "./:/app"
+    env_file:
+      - ./.env_file
+    environment: 
+      ROLE: worker
+      MODEL_NAME: mean_tokens.Model
+    depends_on:
+      elasticmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+  fasttext:
+    build: .
+    platform: linux/amd64
+    volumes:
+      - "./:/app"
+    env_file:
+      - ./.env_file
+    environment: 
+      ROLE: worker
+      MODEL_NAME: fasttext.Model
+    depends_on:
+      elasticmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+  fptg:
+    build: .
+    platform: linux/amd64
+    volumes:
+      - "./:/app"
+    env_file:
+      - ./.env_file
+    environment: 
+      ROLE: worker
+      MODEL_NAME: fptg.Model
+    depends_on:
+      elasticmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+  indian_sbert:
+    build: .
+    platform: linux/amd64
+    volumes:
+      - "./:/app"
+    env_file:
+      - ./.env_file
+    environment: 
+      ROLE: worker
+      MODEL_NAME: indian_sbert.Model
+    depends_on:
+      elasticmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy

requirements.txt

@@ -19,4 +19,5 @@ opentelemetry-api==1.24.0
 opentelemetry-exporter-otlp-proto-http==1.24.0
 opentelemetry-sdk==1.24.0
 redis==5.0.6
-pact-python==2.2.0
+pact-python==2.2.0
+numpy==1.26.4

start_all.sh

@@ -3,19 +3,22 @@
 # Start the first process in the background
 uvicorn main:app --host 0.0.0.0 --port ${PRESTO_PORT} --reload &
 
-# Start the second process in the foreground
-NUM_WORKERS=${NUM_WORKERS:-1}  # Default to 1 worker if not specified
+# Check if ROLE is set to "worker" and start workers if true
+if [ "$ROLE" = "worker" ]; then
+  NUM_WORKERS=${NUM_WORKERS:-1}  # Default to 1 worker if not specified
 
-for i in $(seq 1 $NUM_WORKERS)
-do
-  (
-    while true; do
-      echo "Starting run_worker.py instance $i..."
-      python run_worker.py worker-$i
-      echo "run_worker.py instance $i exited. Restarting..."
-      sleep 30  # Prevent potential rapid restart loop
-    done
-  ) &.  # run workers as background processes
-done
+  for i in $(seq 1 $NUM_WORKERS)
+  do
+    (
+      while true; do
+        echo "Starting run_worker.py instance $i..."
+        python run_worker.py worker-$i
+        echo "run_worker.py instance $i exited. Restarting..."
+        sleep 30  # Prevent potential rapid restart loop
+      done
+    ) &  # run workers as background processes
+  done
+fi
 
+# Start the second process in the foreground
 python run_processor.py