addresses #178

Clearml project and update k8s to new clearml prefix Change DummyBuild to Staging (standard naming) Create staging env for Serval and Echo Remove ClearML_MaxSteps. Remove ASPNETCORE_TranslationEngines params - unneeded
sillsdev · Oct 13, 2023 · c2ff2da · c2ff2da
1 parent 0ee2127
commit c2ff2da
Show file tree

Hide file tree

Showing 14 changed files with 77 additions and 51 deletions.
diff --git a/README.md b/README.md
@@ -27,7 +27,14 @@ Build the Machine repo:
   ```
   dotnet build machine
   ```
-Make sure that the environment variable ASPNETCORE_ENVIRONMENT is set to "Development" by running `export ASPNETCORE_ENVIRONMENT=Development` or adding it to your `.bashrc`.
+Now, if you are using a local docker image and have a GPU with at least 12 GB of RAM:
+* Clone the [Machine repo](https://github.com/sillsdev/machine.py) into an adjacent folder to Serval:
+* Build the docker image with `docker build . -t mpy.local`
+* Register your machine as a ClearML agent (see dev team for details)
+* In docker-compose.yml, update `ClearML__Queue` to be the queue that your computer (agent) is listening to
+If you will use the standard Machine.py images:
+* In docker-compose.yml, update `ClearML__Queue` and `ClearML__DockerImage` to the appropriate values
+
 In the Serval root, run docker compose up
   ```
   cd serval && docker compose up
@@ -70,6 +77,15 @@ There are 3 different environments that Serval is deployed to:
     - `kubectl config use-context aws-rke`
     - `helm upgrade serval deploy/serval -n serval -f deploy/values.yaml`
 
+### Environments:
+- Production:
+  - Full deployment, full NMT and SMT builds
+- Staging:
+  - Full Deploymnet, dummy NMT building (10 steps, small model)
+  - Also used for Docker-compose local staging
+- Development:
+  - Non-docker use only
+
 ## Debugging
 ### To access Serval API
 * Internal QA:

diff --git a/deploy/qa-ext-values.yaml b/deploy/qa-ext-values.yaml
@@ -1,4 +1,5 @@
 externalHost: qa.serval-api.org
+environment: Staging
 enableTls: true
 namespace: serval
 auth0Domain: dev-sillsdev.auth0.com
@@ -7,7 +8,6 @@ lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local
 servalImage: ghcr.io/sillsdev/serval:0.7.1
 machineImage: ghcr.io/sillsdev/machine:3.3.2
 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:0.9.5
-ClearMLMaxSteps: 10
 ClearMLQueue: lambert_24gb
 SharedFileLocation: s3://aqua-ml-data/ext-qa/
 enableEcho: true
diff --git a/deploy/qa-int-values.yaml b/deploy/qa-int-values.yaml
@@ -1,4 +1,5 @@
 externalHost: qa-int.serval-api.org
+environment: Staging
 enableTls: true
 namespace: nlp
 auth0Domain: sil-appbuilder.auth0.com
@@ -7,7 +8,6 @@ lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local
 servalImage: ghcr.io/sillsdev/serval:0.7.1
 machineImage: ghcr.io/sillsdev/machine:3.3.2
 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:0.9.5
-ClearMLMaxSteps: 10
 ClearMLQueue: lambert_24gb
 SharedFileLocation: s3://aqua-ml-data/int-qa/
 enableEcho: true
diff --git a/deploy/serval/templates/echo-deployment.yaml b/deploy/serval/templates/echo-deployment.yaml
@@ -24,16 +24,14 @@ spec:
             - /app/echo_server/EchoTranslationEngine.dll
           workingDir: /app/echo_server
           env:
+            - name: ASPNETCORE_ENVIRONMENT
+              value: {{ .Values.environment}}
             - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Url
               value: http://*:80
             - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols
               value: Http2
             - name: ASPNETCORE_ConnectionStrings__TranslationPlatformApi
               value: http://serval-api:81
-            - name: ASPNETCORE_Logging__LogLevel__Default
-              value: Information
-            - name: ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore
-              value: Warning
           image: {{ .Values.servalImage}}
           imagePullPolicy: "Always"
           name: echo

diff --git a/deploy/serval/templates/machine-engine-deployment.yaml b/deploy/serval/templates/machine-engine-deployment.yaml
@@ -23,6 +23,8 @@ spec:
             - /app/engine_server/SIL.Machine.Serval.EngineServer.dll
           workingDir: /app/engine_server
           env:
+            - name: ASPNETCORE_ENVIRONMENT
+              value: {{ .Values.environment }}
             - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Url
               value: http://*:80
             - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols
@@ -33,14 +35,6 @@ spec:
               value: mongodb://mongo:27017/machine?replicaSet=myRS
             - name: ASPNETCORE_ConnectionStrings__Serval
               value: http://serval-api:81
-            - name: ASPNETCORE_Logging__LogLevel__Default
-              value: Information
-            - name: ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore
-              value: Warning
-            - name: ASPNETCORE_TranslationEngines__0
-              value: SmtTransfer
-            - name: ASPNETCORE_TranslationEngines__1
-              value: Nmt
               {{- template "secrets.clearml" . }}
               {{- template "secrets.s3" . }}
           image: {{ .Values.machineImage}}

diff --git a/deploy/serval/templates/machine-job-deployment.yaml b/deploy/serval/templates/machine-job-deployment.yaml
@@ -23,6 +23,8 @@ spec:
             - /app/job_server/SIL.Machine.Serval.JobServer.dll
           workingDir: /app/job_server
           env:
+            - name: ASPNETCORE_ENVIRONMENT
+              value: {{ .Values.environment}}
             - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Url
               value: http://*:80
             - name: ASPNETCORE_Kestrel__Endpoints__Grpc__Protocols
@@ -33,14 +35,6 @@ spec:
               value: mongodb://mongo:27017/machine?replicaSet=myRS
             - name: ASPNETCORE_ConnectionStrings__Serval
               value: http://serval-api:81
-            - name: ASPNETCORE_Logging__LogLevel__Default
-              value: Information
-            - name: ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore
-              value: Warning
-            - name: ASPNETCORE_TranslationEngines__0
-              value: SmtTransfer
-            - name: ASPNETCORE_TranslationEngines__1
-              value: Nmt
               {{- template "secrets.clearml" . }}
               {{- template "secrets.s3" . }}
           image: {{ .Values.machineImage}}

diff --git a/deploy/serval/templates/secrets.yaml b/deploy/serval/templates/secrets.yaml
@@ -1,18 +1,18 @@
 {{- define "secrets.clearml" }}
-            - name: ClearMLNmtEngine__ApiServer
+            - name: ClearML__ApiServer
               value: https://api.sil.hosted.allegro.ai
-            - name: ClearMLNmtEngine__Queue
+            - name: ClearML__Queue
               value: {{ .Values.ClearMLQueue}}
-            - name: ClearMLNmtEngine__DockerImage
+            - name: ClearML__DockerImage
               value: {{ .Values.ClearMLDockerImage}}
-            - name: ClearMLNmtEngine__MaxSteps
-              value: "{{ .Values.ClearMLMaxSteps}}"
-            - name: ClearMLNmtEngine__AccessKey
+            - name: ClearML__Project
+              value: {{ .Values.externalHost}}
+            - name: ClearML__AccessKey
               valueFrom:
                 secretKeyRef:
                   name: aqua-ml-data
                   key: ClearML_AccessKey
-            - name: ClearMLNmtEngine__SecretKey
+            - name: ClearML__SecretKey
               valueFrom:
                 secretKeyRef:
                   name: aqua-ml-data

diff --git a/deploy/serval/templates/serval-api-deployment.yaml b/deploy/serval/templates/serval-api-deployment.yaml
@@ -25,6 +25,8 @@ spec:
           env:
             - name: Auth__Domain
               value: {{ .Values.auth0Domain}}
+            - name: ASPNETCORE_ENVIRONMENT
+              value: {{ .Values.environment}}
             - name: ASPNETCORE_Kestrel__Endpoints__Http__Url
               value: http://*:80
             - name: ASPNETCORE_Kestrel__Endpoints__Http2__Url
@@ -35,10 +37,6 @@ spec:
               value: mongodb://mongo:27017/serval_jobs?replicaSet=myRS
             - name: ASPNETCORE_ConnectionStrings__Mongo
               value: mongodb://mongo:27017/serval?replicaSet=myRS
-            - name: ASPNETCORE_Logging__LogLevel__Default
-              value: Information
-            - name: ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore
-              value: Warning
             - name: ASPNETCORE_Translation__Engines__0__Type
               value: Echo
             - name: ASPNETCORE_Translation__Engines__0__Address

diff --git a/deploy/values.yaml b/deploy/values.yaml
@@ -1,4 +1,5 @@
 externalHost: prod.serval-api.org
+environment: Production
 enableTls: true
 namespace: serval
 auth0Domain: languagetechnology.auth0.com
@@ -7,7 +8,6 @@ lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local
 servalImage: ghcr.io/sillsdev/serval:0.7.1
 machineImage: ghcr.io/sillsdev/machine:3.3.2
 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:0.9.5
-ClearMLMaxSteps: 20000
 ClearMLQueue: production
 SharedFileLocation: s3://aqua-ml-data/production/
 enableEcho: true
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -7,15 +7,14 @@ services:
       context: .
       dockerfile: dockerfile.development
     environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
       - Auth__Domain=sil-appbuilder.auth0.com
       - Auth__Audience=https://machine.sil.org
       - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
       - ASPNETCORE_Kestrel__Endpoints__Http2__Url=http://*:81
       - ASPNETCORE_Kestrel__Endpoints__Http2__Protocols=Http2
       - ASPNETCORE_ConnectionStrings__Hangfire=mongodb://mongo:27017/serval_jobs?replicaSet=myRS
       - ASPNETCORE_ConnectionStrings__Mongo=mongodb://mongo:27017/serval?replicaSet=myRS
-      - ASPNETCORE_Logging__LogLevel__Default=Information
-      - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning
       - ASPNETCORE_Translation__Engines__0__Type=Echo
       - ASPNETCORE_Translation__Engines__0__Address=http://echo
       - ASPNETCORE_Translation__Engines__1__Type=SmtTransfer
@@ -49,10 +48,9 @@ services:
       context: .
       dockerfile: dockerfile.development
     environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
       - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
       - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
-      - ASPNETCORE_Logging__LogLevel__Default=Information
-      - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning
       - ASPNETCORE_ConnectionStrings__TranslationPlatformApi=http://serval-api:81
     expose:
       - 80
@@ -81,18 +79,16 @@ services:
       dockerfile: ../machine/dockerfile.development
 
     environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
       - ASPNETCORE_Kestrel__Endpoints__Https__Url=http://*:80
       - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
       - ASPNETCORE_ConnectionStrings__Hangfire=mongodb://mongo:27017/machine_jobs?replicaSet=myRS
       - ASPNETCORE_ConnectionStrings__Mongo=mongodb://mongo:27017/machine?replicaSet=myRS
       - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81
-      - ASPNETCORE_Logging__LogLevel__Default=Information
-      - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning
-      - ASPNETCORE_TranslationEngines__0=SmtTransfer
-      - ASPNETCORE_TranslationEngines__1=Nmt
       - ClearML__ApiServer=https://api.sil.hosted.allegro.ai
-      - ClearML__Queue=production
-      - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5.1
+      - ClearML__Queue=lambert_24gb
+      - ClearML__DockerImage=mpy.local
+      - ClearML__Project=docker-compose
       - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}"
       - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}"
       - SharedFile__Uri=s3://aqua-ml-data/docker-compose/
@@ -126,18 +122,16 @@ services:
       context: ../machine
       dockerfile: ../machine/dockerfile.development
     environment:
+      - ASPNETCORE_ENVIRONMENT=Staging
       - ASPNETCORE_ConnectionStrings__Hangfire=mongodb://mongo:27017/machine_jobs?replicaSet=myRS
       - ASPNETCORE_ConnectionStrings__Mongo=mongodb://mongo:27017/machine?replicaSet=myRS
       - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81
-      - ASPNETCORE_Logging__LogLevel__Default=Information
-      - ASPNETCORE_Logging__LogLevel__Microsoft.AspNetCore=Warning
       - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
       - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
-      - ASPNETCORE_TranslationEngines__0=SmtTransfer
-      - ASPNETCORE_TranslationEngines__1=Nmt
       - ClearML__ApiServer=https://api.sil.hosted.allegro.ai
-      - ClearML__Queue=production
-      - ClearML__DockerImage=ghcr.io/sillsdev/machine.py:0.9.5.1
+      - ClearML__Queue=lambert_24gb
+      - ClearML__DockerImage=mpy.local
+      - ClearML__Project=docker-compose
       - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}"
       - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}"
       - SharedFile__Uri=s3://aqua-ml-data/docker-compose/

diff --git a/samples/EchoTranslationEngine/appsettings.Production.json b/samples/EchoTranslationEngine/appsettings.Production.json
@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}
diff --git a/samples/EchoTranslationEngine/appsettings.Staging.json b/samples/EchoTranslationEngine/appsettings.Staging.json
@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}
diff --git a/src/Serval.ApiServer/appsettings.Production.json b/src/Serval.ApiServer/appsettings.Production.json
@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}
diff --git a/src/Serval.ApiServer/appsettings.Staging.json b/src/Serval.ApiServer/appsettings.Staging.json
@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}