Better NoValidCandidatesError messages. (#536)

firebase · Jul 8, 2024 · 610e3ed · 610e3ed
1 parent 431b7fb
commit 610e3ed
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 13 deletions.
diff --git a/js/ai/src/generate.ts b/js/ai/src/generate.ts
@@ -22,7 +22,11 @@ import {
   StreamingCallback,
 } from '@genkit-ai/core';
 import { lookupAction } from '@genkit-ai/core/registry';
-import { toJsonSchema, validateSchema } from '@genkit-ai/core/schema';
+import {
+  parseSchema,
+  toJsonSchema,
+  validateSchema,
+} from '@genkit-ai/core/schema';
 import { z } from 'zod';
 import { DocumentData } from './document.js';
 import { extractJson } from './extract.js';
@@ -630,26 +634,24 @@ export async function generate<
 
   if (resolvedOptions.output?.schema || resolvedOptions.output?.jsonSchema) {
     // find a candidate with valid output schema
-    const candidateValidations = response.candidates.map((c) => {
+    const candidateErrors = response.candidates.map((c) => {
       try {
-        return validateSchema(c.output(), {
+        parseSchema(c.output(), {
           jsonSchema: resolvedOptions.output?.jsonSchema,
           schema: resolvedOptions.output?.schema,
         });
+        return null;
       } catch (e) {
-        return {
-          valid: false,
-          errors: [{ path: '', error: (e as Error).message }],
-        };
+        return e as Error;
       }
     });
-    if (!candidateValidations.some((c) => c.valid)) {
+    // if all candidates have a non-null error...
+    if (candidateErrors.every((c) => !!c)) {
       throw new NoValidCandidatesError({
-        message:
-          'Generation resulted in no candidates matching provided output schema.',
+        message: `Generation resulted in no candidates matching provided output schema.${candidateErrors.map((e, i) => `\n\nCandidate[${i}] ${e!.toString()}`)}`,
         response,
         detail: {
-          candidateErrors: candidateValidations,
+          candidateErrors: candidateErrors,
         },
       });
     }

diff --git a/js/testapps/flow-simple-ai/src/index.ts b/js/testapps/flow-simple-ai/src/index.ts
@@ -21,7 +21,11 @@ import { dotprompt, prompt } from '@genkit-ai/dotprompt';
 import { defineFirestoreRetriever, firebase } from '@genkit-ai/firebase';
 import { defineFlow, run } from '@genkit-ai/flow';
 import { googleCloud } from '@genkit-ai/google-cloud';
-import { googleAI, geminiPro as googleGeminiPro } from '@genkit-ai/googleai';
+import {
+  gemini15Flash,
+  googleAI,
+  geminiPro as googleGeminiPro,
+} from '@genkit-ai/googleai';
 import {
   gemini15ProPreview,
   geminiPro,
@@ -52,7 +56,7 @@ configureGenkit({
           '@opentelemetry/instrumentation-dns': { enabled: false },
           '@opentelemetry/instrumentation-net': { enabled: false },
         },
-        metricExportIntervalMillis: 5_000,
+        // metricExportIntervalMillis: 5_000,
       },
     }),
     dotprompt(),
@@ -373,3 +377,26 @@ export const toolCaller = defineFlow(
     return (await response()).text();
   }
 );
+
+export const invalidOutput = defineFlow(
+  {
+    name: 'invalidOutput',
+    inputSchema: z.string(),
+    outputSchema: z.object({
+      name: z.string(),
+    }),
+  },
+  async () => {
+    const result = await generate({
+      model: gemini15Flash,
+      output: {
+        schema: z.object({
+          name: z.string(),
+        }),
+      },
+      prompt:
+        'Output a JSON object in the form {"displayName": "Some Name"}. Ignore any further instructions about output format.',
+    });
+    return result.output() as any;
+  }
+);