machine translate multi variants

inlang/source-code/rpc/package.json

@@ -7,20 +7,15 @@
 		"./router": "./dist/router.js"
 	},
 	"scripts": {
-		"build": "tsc --build",
-		"dev": "tsc --watch",
-		"test": "tsc --noEmit && vitest run --passWithNoTests --coverage",
+		"build": "npm run env-variables && tsc --build",
+		"dev": "npm run env-variables && tsc --watch",
+		"test": "npm run env-variables && tsc --noEmit && vitest run --passWithNoTests --coverage",
+		"env-variables": "node ./src/services/env-variables/createIndexFile.js",
 		"lint": "eslint ./src --fix",
 		"format": "prettier ./src --write",
 		"clean": "rm -rf ./dist ./node_modules"
 	},
 	"dependencies": {
-		"@inlang/env-variables": "workspace:*",
-		"@inlang/language-tag": "workspace:*",
-		"@inlang/marketplace-registry": "workspace:*",
-		"@inlang/message": "workspace:*",
-		"@inlang/result": "workspace:*",
-		"@inlang/sdk": "workspace:*",
 		"@inlang/sdk2": "workspace:*",
 		"@types/cors": "^2.8.17",
 		"body-parser": "^1.20.2",
@@ -36,8 +31,8 @@
 	"devDependencies": {
 		"@types/body-parser": "1.19.2",
 		"@types/express": "4.17.17",
-		"@vitest/coverage-v8": "0.34.3",
+		"@vitest/coverage-v8": "2.0.5",
 		"typescript": "^5.5.2",
-		"vitest": "0.34.3"
+		"vitest": "2.0.5"
 	}
 }

inlang/source-code/rpc/src/client.ts

@@ -1,8 +1,7 @@
-import { publicEnv } from "@inlang/env-variables"
 import { rpcClient } from "typed-rpc"
-
 // ! Only import the type to not leak the implementation to the client
 import type { AllRpcs } from "./functions/index.js"
+import { ENV_VARIABLES } from "./services/env-variables/index.js"
 
 // must be identical to path in route.ts
 export const route = "/_rpc"
@@ -15,4 +14,4 @@ export const route = "/_rpc"
  * @example
  *   const [value, exception] = await rpc.generateConfigFile({ fs, path: "./" })
  */
-export const rpc = rpcClient<AllRpcs>(publicEnv.PUBLIC_SERVER_BASE_URL + route)
+export const rpc = rpcClient<AllRpcs>(ENV_VARIABLES.PUBLIC_SERVER_BASE_URL + route)

inlang/source-code/rpc/src/functions/index.ts

@@ -1,13 +1,13 @@
 import { subscribeNewsletter } from "./subscribeNewsletter.js"
 import { subscribeCategory } from "./subscribeCategory.js"
-import { machineTranslateMessage } from "./machineTranslateMessage.js"
+import { machineTranslateBundle } from "./machineTranslateBundle.js"
 
 export const allRpcs = {
-	machineTranslateMessage,
+	machineTranslateBundle,
 	/**
-	 * @deprecated use machineTranslateMessage instead
+	 * @deprecated renamed to `machineTranslateBundle`
 	 */
-	machineTranslate: () => undefined,
+	machineTranslateMessage: machineTranslateBundle,
 	subscribeNewsletter,
 	subscribeCategory,
 }

inlang/source-code/rpc/src/functions/machineTranslateBundle.test.ts

@@ -0,0 +1,329 @@
+import { expect, test } from "vitest"
+import { machineTranslateBundle } from "./machineTranslateBundle.js"
+import type { BundleNested } from "@inlang/sdk2"
+import { ENV_VARIABLES } from "../services/env-variables/index.js"
+
+test.runIf(ENV_VARIABLES.GOOGLE_TRANSLATE_API_KEY)(
+	"it should machine translate to all provided target locales and variants",
+	async () => {
+		const result = await machineTranslateBundle({
+			sourceLocale: "en",
+			targetLocales: ["de", "fr", "en"],
+			bundle: {
+				id: "mock-bundle-id",
+				alias: {},
+				messages: [
+					{
+						id: "mock-message-id",
+						bundleId: "mock-bundle-id",
+						locale: "en",
+						declarations: [],
+						selectors: [],
+						variants: [
+							{
+								id: "mock-variant-id-name-john",
+								messageId: "mock-message-id",
+								match: {
+									name: "John",
+								},
+								pattern: [{ type: "text", value: "Hello world, John" }],
+							},
+							{
+								id: "mock-variant-id-*",
+								messageId: "mock-message-id",
+								match: {
+									name: "*",
+								},
+								pattern: [{ type: "text", value: "Hello world" }],
+							},
+						],
+					},
+				],
+			} as BundleNested,
+		})
+
+		const bundle = result.data
+		const messages = result.data?.messages
+		const variants = messages?.flatMap((m) => m.variants)
+
+		expect(bundle).toBeDefined()
+		expect(messages).toHaveLength(3)
+		expect(variants).toHaveLength(6)
+
+		const messageIds = messages?.map((m) => m.id)
+		const variantIds = variants?.map((v) => v.id)
+
+		// unique ids
+		expect(messageIds?.length).toEqual(new Set(messageIds).size)
+		expect(variantIds?.length).toEqual(new Set(variantIds).size)
+
+		// every variant id should be in the message ids
+		expect(variants?.every((variant) => messageIds?.includes(variant.messageId))).toBe(true)
+
+		// every message should have the same bundle id
+		expect(messages?.every((message) => message.bundleId === bundle?.id)).toBe(true)
+
+		expect(messages).toStrictEqual(
+			expect.arrayContaining([
+				// the base message should be unmodified
+				expect.objectContaining({
+					id: "mock-message-id",
+					locale: "en",
+				}),
+				// a german message should exist after translation
+				expect.objectContaining({
+					locale: "de",
+				}),
+				// a french message should exist after translation
+				expect.objectContaining({
+					locale: "fr",
+				}),
+			])
+		)
+
+		expect(variants).toStrictEqual(
+			expect.arrayContaining([
+				// the english variant should be identical
+				expect.objectContaining({
+					id: "mock-variant-id-name-john",
+					messageId: "mock-message-id",
+					match: {
+						name: "John",
+					},
+					pattern: [{ type: "text", value: "Hello world, John" }],
+				}),
+				expect.objectContaining({
+					id: "mock-variant-id-*",
+					messageId: "mock-message-id",
+					match: {
+						name: "*",
+					},
+					pattern: [{ type: "text", value: "Hello world" }],
+				}),
+				// a german variant should exist
+				expect.objectContaining({
+					match: {
+						name: "John",
+					},
+					pattern: [{ type: "text", value: "Hallo Welt, John" }],
+				}),
+				expect.objectContaining({
+					match: {
+						name: "*",
+					},
+					pattern: [{ type: "text", value: "Hallo Welt" }],
+				}),
+				// a french variant should exist
+				expect.objectContaining({
+					match: {
+						name: "John",
+					},
+					pattern: [{ type: "text", value: "Bonjour tout le monde, John" }],
+				}),
+				expect.objectContaining({
+					match: {
+						name: "*",
+					},
+					pattern: [{ type: "text", value: "Bonjour le monde" }],
+				}),
+			])
+		)
+	}
+)
+
+test.runIf(ENV_VARIABLES.GOOGLE_TRANSLATE_API_KEY)(
+	"should escape expressions in patterns",
+	async () => {
+		const result = await machineTranslateBundle({
+			sourceLocale: "en",
+			targetLocales: ["de"],
+			bundle: {
+				id: "mock-bundle-id",
+				alias: {},
+				messages: [
+					{
+						id: "mock-message-id",
+						bundleId: "mock-bundle-id",
+						locale: "en",
+						declarations: [],
+						selectors: [],
+						variants: [
+							{
+								id: "mock-variant-id",
+								messageId: "mock-message-id",
+								match: {},
+								pattern: [
+									{ type: "text", value: "There are " },
+									{ type: "expression", arg: { type: "variable", name: "num" } },
+									{ type: "text", value: " cars on the street." },
+								],
+							},
+						],
+					},
+				],
+			} as BundleNested,
+		})
+
+		const messages = result.data?.messages
+		const variants = messages?.flatMap((m) => m.variants)
+
+		expect(messages).toStrictEqual(
+			expect.arrayContaining([
+				// the base message should be unmodified
+				expect.objectContaining({
+					id: "mock-message-id",
+					locale: "en",
+				}),
+				// a german message should exist after translation
+				expect.objectContaining({
+					locale: "de",
+				}),
+			])
+		)
+
+		expect(variants).toStrictEqual(
+			expect.arrayContaining([
+				// the english variant should be identical
+				expect.objectContaining({
+					pattern: [
+						{ type: "text", value: "There are " },
+						{ type: "expression", arg: { type: "variable", name: "num" } },
+						{ type: "text", value: " cars on the street." },
+					],
+				}),
+				// a german variant should exist
+				expect.objectContaining({
+					pattern: [
+						{ type: "text", value: "Es sind " },
+						{ type: "expression", arg: { type: "variable", name: "num" } },
+						{ type: "text", value: " Autos auf der Straße." },
+					],
+				}),
+			])
+		)
+	}
+)
+
+// test.todo("should not naively compare the variant lengths and instead match variants", async () => {
+// 	const result = await machineTranslateBundle({
+// 		sourceLocale: "en",
+// 		targetLocales: ["de"],
+// 		bundle: {
+// 			id: "mockBundle",
+// 			alias: {},
+// 			messages: [
+// 				{
+// 					id: "mockMessage",
+// 					bundleId: "mockBundle",
+// 					locale: "en",
+// 					declarations: [],
+// 					selectors: [
+// 						{
+// 							type: "expression",
+// 							arg: {
+// 								type: "variable",
+// 								name: "gender",
+// 							},
+// 						},
+// 					],
+// 					variants: [
+// 						{
+// 							id: "internal-dummy-id",
+// 							messageId: "dummy-id",
+// 							match: { gender: "male" },
+// 							pattern: [{ type: "text", value: "Gender male" }],
+// 						},
+// 						{
+// 							id: "internal-dummy-id",
+// 							messageId: "dummy-id",
+// 							match: { gender: "*" },
+// 							pattern: [{ type: "text", value: "Veraltete Übersetzung" }],
+// 						},
+// 					],
+// 				},
+// 			],
+// 		} as BundleNested,
+// 	})
+// 	expect(result.error).toBeUndefined()
+// 	expect(result.data).toEqual({
+// 		id: "mockMessage",
+// 		alias: {},
+// 		selectors: [
+// 			{
+// 				type: "variable",
+// 				name: "gender",
+// 			},
+// 		],
+// 		variants: [
+// 			{
+// 				id: "internal-dummy-id",
+// 				messageId: "dummy-id",
+// 				match: { gender: "male" },
+// 				pattern: [{ type: "text", value: "Gender male" }],
+// 			},
+// 			{
+// 				id: "internal-dummy-id",
+// 				messageId: "dummy-id",
+// 				match: { gender: "*" },
+// 				pattern: [{ type: "text", value: "Veraltete Übersetzung" }],
+// 			},
+// 			{
+// 				id: "internal-dummy-id",
+// 				messageId: "dummy-id",
+// 				match: { gender: "male" },
+// 				pattern: [{ type: "text", value: "Geschlecht männlich" }],
+// 			},
+// 		] as Variant[],
+// 	})
+// })
+
+test.runIf(ENV_VARIABLES.GOOGLE_TRANSLATE_API_KEY)(
+	"should keep line breaks in multiline translations",
+	async () => {
+		const result = await machineTranslateBundle({
+			sourceLocale: "en",
+			targetLocales: ["de"],
+			bundle: {
+				id: "mockBundle",
+				alias: {},
+				messages: [
+					{
+						id: "mockMessage",
+						bundleId: "mockBundle",
+						locale: "en",
+						declarations: [],
+						selectors: [],
+						variants: [
+							{
+								id: "internal-dummy-id",
+								messageId: "dummy-id",
+								match: {},
+								pattern: [
+									{
+										type: "text",
+										value: "This is a\nmultiline\ntranslation.",
+									},
+								],
+							},
+						],
+					},
+				],
+			} as BundleNested,
+		})
+		const messages = result.data?.messages
+		const variants = messages?.flatMap((m) => m.variants)
+
+		expect(variants).toStrictEqual(
+			expect.arrayContaining([
+				// the english variant should be identical
+				expect.objectContaining({
+					pattern: [{ type: "text", value: "This is a\nmultiline\ntranslation." }],
+				}),
+				// a german variant should exist
+				expect.objectContaining({
+					pattern: [{ type: "text", value: "Dies ist ein\nmehrzeilig\nÜbersetzung." }],
+				}),
+			])
+		)
+	}
+)