[AI] API for defining pools of AI related endpoints (#10100)

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> Co-authored-by: Shashank Ram <21697719+shashankram@users.noreply.github.com>
solo-io · Sep 26, 2024 · 2a290f0 · 2a290f0
1 parent c46ed68
commit 2a290f0
Show file tree

Hide file tree

Showing 22 changed files with 2,138 additions and 452 deletions.
diff --git a/changelog/v1.18.0-beta23/ai-prioritized-endpoints.yaml b/changelog/v1.18.0-beta23/ai-prioritized-endpoints.yaml
@@ -0,0 +1,5 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/solo-projects/issues/6957
+    description: >- 
+      Add an API to allow configuring prioritized pools of LLM backends.
diff --git a/...thub.com/solo-io/gloo/projects/gloo/api/v1/enterprise/options/ai/ai.proto.sk.md b/...thub.com/solo-io/gloo/projects/gloo/api/v1/enterprise/options/ai/ai.proto.sk.md
diff --git a/...ithub.com/solo-io/gloo/projects/gloo/api/v1/options/retries/retries.proto.sk.md b/...ithub.com/solo-io/gloo/projects/gloo/api/v1/options/retries/retries.proto.sk.md
diff --git a/install/helm/gloo/crds/gateway.solo.io_v1_RouteOption.yaml b/install/helm/gloo/crds/gateway.solo.io_v1_RouteOption.yaml
@@ -35,10 +35,6 @@ spec:
                 properties:
                   ai:
                     properties:
-                      backupModels:
-                        items:
-                          type: string
-                        type: array
                       defaults:
                         items:
                           properties:
@@ -149,6 +145,9 @@ spec:
                           promptTemplate:
                             type: string
                         type: object
+                      routeType:
+                        type: string
+                        x-kubernetes-int-or-string: true
                       semanticCache:
                         properties:
                           datastore:
@@ -1573,6 +1572,12 @@ spec:
                             nullable: true
                             type: integer
                         type: object
+                      retriableStatusCodes:
+                        items:
+                          maximum: 4294967295
+                          minimum: 0
+                          type: integer
+                        type: array
                       retryBackOff:
                         properties:
                           baseInterval:

diff --git a/install/helm/gloo/crds/gateway.solo.io_v1_RouteTable.yaml b/install/helm/gloo/crds/gateway.solo.io_v1_RouteTable.yaml
@@ -145,10 +145,6 @@ spec:
                       properties:
                         ai:
                           properties:
-                            backupModels:
-                              items:
-                                type: string
-                              type: array
                             defaults:
                               items:
                                 properties:
@@ -259,6 +255,9 @@ spec:
                                 promptTemplate:
                                   type: string
                               type: object
+                            routeType:
+                              type: string
+                              x-kubernetes-int-or-string: true
                             semanticCache:
                               properties:
                                 datastore:
@@ -1683,6 +1682,12 @@ spec:
                                   nullable: true
                                   type: integer
                               type: object
+                            retriableStatusCodes:
+                              items:
+                                maximum: 4294967295
+                                minimum: 0
+                                type: integer
+                              type: array
                             retryBackOff:
                               properties:
                                 baseInterval:

diff --git a/install/helm/gloo/crds/gateway.solo.io_v1_VirtualHostOption.yaml b/install/helm/gloo/crds/gateway.solo.io_v1_VirtualHostOption.yaml
@@ -1342,6 +1342,12 @@ spec:
                             nullable: true
                             type: integer
                         type: object
+                      retriableStatusCodes:
+                        items:
+                          maximum: 4294967295
+                          minimum: 0
+                          type: integer
+                        type: array
                       retryBackOff:
                         properties:
                           baseInterval: