diff --git a/js/ai/src/generate.ts b/js/ai/src/generate.ts index d4718772b..f423df019 100755 --- a/js/ai/src/generate.ts +++ b/js/ai/src/generate.ts @@ -22,7 +22,11 @@ import { StreamingCallback, } from '@genkit-ai/core'; import { lookupAction } from '@genkit-ai/core/registry'; -import { toJsonSchema, validateSchema } from '@genkit-ai/core/schema'; +import { + parseSchema, + toJsonSchema, + validateSchema, +} from '@genkit-ai/core/schema'; import { z } from 'zod'; import { DocumentData } from './document.js'; import { extractJson } from './extract.js'; @@ -630,26 +634,24 @@ export async function generate< if (resolvedOptions.output?.schema || resolvedOptions.output?.jsonSchema) { // find a candidate with valid output schema - const candidateValidations = response.candidates.map((c) => { + const candidateErrors = response.candidates.map((c) => { try { - return validateSchema(c.output(), { + parseSchema(c.output(), { jsonSchema: resolvedOptions.output?.jsonSchema, schema: resolvedOptions.output?.schema, }); + return null; } catch (e) { - return { - valid: false, - errors: [{ path: '', error: (e as Error).message }], - }; + return e as Error; } }); - if (!candidateValidations.some((c) => c.valid)) { + // if all candidates have a non-null error... + if (candidateErrors.every((c) => !!c)) { throw new NoValidCandidatesError({ - message: - 'Generation resulted in no candidates matching provided output schema.', + message: `Generation resulted in no candidates matching provided output schema.${candidateErrors.map((e, i) => `\n\nCandidate[${i}] ${e!.toString()}`)}`, response, detail: { - candidateErrors: candidateValidations, + candidateErrors: candidateErrors, }, }); } diff --git a/js/testapps/flow-simple-ai/src/index.ts b/js/testapps/flow-simple-ai/src/index.ts index ac6ccff94..3cb1f17fd 100644 --- a/js/testapps/flow-simple-ai/src/index.ts +++ b/js/testapps/flow-simple-ai/src/index.ts @@ -21,7 +21,11 @@ import { dotprompt, prompt } from '@genkit-ai/dotprompt'; import { defineFirestoreRetriever, firebase } from '@genkit-ai/firebase'; import { defineFlow, run } from '@genkit-ai/flow'; import { googleCloud } from '@genkit-ai/google-cloud'; -import { googleAI, geminiPro as googleGeminiPro } from '@genkit-ai/googleai'; +import { + gemini15Flash, + googleAI, + geminiPro as googleGeminiPro, +} from '@genkit-ai/googleai'; import { gemini15ProPreview, geminiPro, @@ -52,7 +56,7 @@ configureGenkit({ '@opentelemetry/instrumentation-dns': { enabled: false }, '@opentelemetry/instrumentation-net': { enabled: false }, }, - metricExportIntervalMillis: 5_000, + // metricExportIntervalMillis: 5_000, }, }), dotprompt(), @@ -373,3 +377,26 @@ export const toolCaller = defineFlow( return (await response()).text(); } ); + +export const invalidOutput = defineFlow( + { + name: 'invalidOutput', + inputSchema: z.string(), + outputSchema: z.object({ + name: z.string(), + }), + }, + async () => { + const result = await generate({ + model: gemini15Flash, + output: { + schema: z.object({ + name: z.string(), + }), + }, + prompt: + 'Output a JSON object in the form {"displayName": "Some Name"}. Ignore any further instructions about output format.', + }); + return result.output() as any; + } +);