From 815e029a73161fb217f6e811dd3824d4f45b18b8 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:41:07 -0400 Subject: [PATCH 01/11] refactor: dedicated export --- .../i18next/src/import-export/exportFiles.ts | 10 +- .../i18next/src/import-export/importFiles.ts | 70 ++-- .../src/import-export/roundtrip.test.ts | 344 +++++++++--------- .../src/import-export/importFiles.test.ts | 56 +++ .../sdk2/src/import-export/importFiles.ts | 100 +++++ .../sdk2/src/import-export/index.ts | 43 +-- .../sdk2/src/import-export/roundtrip.test.ts | 209 +++++------ inlang/source-code/sdk2/src/plugin/errors.ts | 2 +- inlang/source-code/sdk2/src/plugin/schema.ts | 61 +++- .../sdk2/src/project/loadProject.ts | 3 +- .../project/loadProjectFromDirectory.test.ts | 6 +- .../project/saveProjectToDirectory.test.ts | 60 +-- .../src/project/saveProjectToDirectory.ts | 22 +- 13 files changed, 586 insertions(+), 400 deletions(-) create mode 100644 inlang/source-code/sdk2/src/import-export/importFiles.test.ts create mode 100644 inlang/source-code/sdk2/src/import-export/importFiles.ts diff --git a/inlang/source-code/plugins/i18next/src/import-export/exportFiles.ts b/inlang/source-code/plugins/i18next/src/import-export/exportFiles.ts index 0db8bc9dc5..4a26ce6346 100644 --- a/inlang/source-code/plugins/i18next/src/import-export/exportFiles.ts +++ b/inlang/source-code/plugins/i18next/src/import-export/exportFiles.ts @@ -3,17 +3,19 @@ import type { Bundle, LiteralMatch, Message, Pattern, Variant } from "@inlang/sd import { type plugin } from "../plugin.js" import { unflatten } from "flat" -export const exportFiles: NonNullable<(typeof plugin)["exportFiles"]> = async ({ bundles }) => { +export const exportFiles: NonNullable<(typeof plugin)["exportFiles"]> = async ({ + bundles, + messages, + variants, +}) => { const result: Record> = {} const resultNamespaces: Record>> = {} - const messages = bundles.flatMap((bundle) => bundle.messages) - for (const message of messages) { const serializedMessages = serializeMessage( bundles.find((b) => b.id === message.bundleId)!, message, - message.variants + variants.filter((v) => v.messageId === message.id) ) for (const message of serializedMessages) { diff --git a/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts b/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts index 8dc11ac435..09bd82275b 100644 --- a/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts +++ b/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts @@ -1,20 +1,17 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ -import type { - Bundle, - BundleNested, - Message, - Pattern, - VariableReference, - Variant, -} from "@inlang/sdk2" +import type { Pattern, VariableReference, Variant } from "@inlang/sdk2" import { type plugin } from "../plugin.js" import { flatten } from "flat" +import type { + BundleImport, + MessageImport, + VariantImport, +} from "../../../../sdk2/dist/plugin/schema.js" export const importFiles: NonNullable<(typeof plugin)["importFiles"]> = async ({ files }) => { - const result: BundleNested[] = [] - const bundles: Bundle[] = [] - const messages: Message[] = [] - const variants: Variant[] = [] + const bundles: BundleImport[] = [] + const messages: MessageImport[] = [] + const variants: VariantImport[] = [] for (const file of files) { const namespace = file.toBeImportedFilesMetadata?.namespace @@ -26,41 +23,27 @@ export const importFiles: NonNullable<(typeof plugin)["importFiles"]> = async ({ // merge the bundle declarations const uniqueBundleIds = [...new Set(bundles.map((bundle) => bundle.id))] - const uniqueBundles: Bundle[] = uniqueBundleIds.map((id) => { + const uniqueBundles: BundleImport[] = uniqueBundleIds.map((id) => { const _bundles = bundles.filter((bundle) => bundle.id === id) const declarations = removeDuplicates(_bundles.flatMap((bundle) => bundle.declarations)) return { id, declarations } }) - // establishing nesting - for (const bundle of uniqueBundles) { - const bundleNested: BundleNested = { ...bundle, messages: [] } - - // @ts-expect-error - casting the type here - bundleNested.messages = messages.filter((message) => message.bundleId === bundle.id) - - for (const message of bundleNested.messages) { - message.variants = variants.filter((variant) => variant.messageId === message.id) - } - - result.push(bundleNested) - } - - return { bundles: result } + return { bundles: uniqueBundles, messages, variants } } function parseFile(args: { namespace?: string; locale: string; content: ArrayBuffer }): { - bundles: Bundle[] - messages: Message[] - variants: Variant[] + bundles: BundleImport[] + messages: MessageImport[] + variants: VariantImport[] } { const resource: Record = flatten( JSON.parse(new TextDecoder().decode(args.content)) ) - const bundles: Bundle[] = [] - const messages: Message[] = [] - const variants: Variant[] = [] + const bundles: BundleImport[] = [] + const messages: MessageImport[] = [] + const variants: VariantImport[] = [] for (const key in resource) { const value = resource[key]! @@ -84,7 +67,7 @@ function parseMessage(args: { value: string locale: string resource: Record -}): { bundle: Bundle; message: Message; variant: Variant } { +}): { bundle: BundleImport; message: MessageImport; variant: VariantImport } { const pattern = parsePattern(args.value) // i18next suffixes keys with context or plurals @@ -96,7 +79,7 @@ function parseMessage(args: { bundleId = `${args.namespace}:${bundleId}` } - const bundle: Bundle = { + const bundle: BundleImport = { id: bundleId, declarations: pattern.variableReferences.map((variableReference) => ({ type: "input-variable", @@ -104,16 +87,15 @@ function parseMessage(args: { })), } - const message: Message = { - id: "", + const message: MessageImport = { bundleId: bundleId, selectors: [], locale: args.locale, } - const variant: Variant = { - id: "", - messageId: "", + const variant: VariantImport = { + bundleId: bundleId, + locale: args.locale, matches: [], pattern: pattern.result, } @@ -259,12 +241,6 @@ function parseMessage(args: { bundle.declarations = removeDuplicates(bundle.declarations) - // i18next suffixes keys with context or plurals - // "friend_female_one" -> "friend" - message.id = `${bundle.id};;locale=${args.locale};;` - variant.id = `${message.id};;match=${variant.matches.join(",")};;` - variant.messageId = message.id - return { bundle, message, variant } } diff --git a/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts b/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts index ae234506a5..1f0f2cac04 100644 --- a/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts +++ b/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts @@ -1,70 +1,68 @@ import { expect, test } from "vitest" import { importFiles } from "./importFiles.js" -import type { BundleNested, LiteralMatch, Pattern, Variant } from "@inlang/sdk2" +import { + type Bundle, + type LiteralMatch, + type Message, + type Pattern, + type Variant, +} from "@inlang/sdk2" import { exportFiles } from "./exportFiles.js" test("single key value", async () => { const imported = await runImportFiles({ key: "value", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ key: "value", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "key") + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(1) - expect(bundle?.messages[0]?.selectors).toStrictEqual([]) - expect(bundle?.messages[0]?.variants[0]?.matches).toStrictEqual([]) - - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "value" }, - ]) + expect(imported.bundles[0]?.id).toStrictEqual("key") + expect(imported.bundles[0]?.declarations).toStrictEqual([]) + expect(imported.messages[0]?.selectors).toStrictEqual([]) + expect(imported.variants[0]?.matches).toStrictEqual([]) + expect(imported.variants[0]?.pattern).toStrictEqual([{ type: "text", value: "value" }]) }) test("key deep", async () => { const imported = await runImportFiles({ - keyDeep: { - inner: "value", - }, + keyDeep: { inner: "value" }, }) - expect(await runExportFiles(imported)).toStrictEqual({ - keyDeep: { - inner: "value", - }, + expect(await runExportFilesParsed(imported)).toStrictEqual({ + keyDeep: { inner: "value" }, }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyDeep.inner") - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "value" }, - ]) + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(1) + + expect(imported.bundles[0]?.id).toStrictEqual("keyDeep.inner") + expect(imported.variants[0]?.pattern).toStrictEqual([{ type: "text", value: "value" }]) }) test("keyInterpolate", async () => { const imported = await runImportFiles({ keyInterpolate: "replace this {{value}}", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyInterpolate: "replace this {{value}}", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyInterpolate") + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(1) - expect(bundle?.declarations).toStrictEqual([ - { - type: "input-variable", - name: "value", - }, + expect(imported.bundles[0]?.declarations).toStrictEqual([ + { type: "input-variable", name: "value" }, ]) - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ + expect(imported.variants[0]?.pattern).toStrictEqual([ { type: "text", value: "replace this " }, - { - type: "expression", - arg: { - type: "variable-reference", - name: "value", - }, - }, + { type: "expression", arg: { type: "variable-reference", name: "value" } }, ] satisfies Pattern) }) @@ -72,26 +70,17 @@ test("keyInterpolateUnescaped", async () => { const imported = await runImportFiles({ keyInterpolateUnescaped: "replace this {{- value}}", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyInterpolateUnescaped: "replace this {{- value}}", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyInterpolateUnescaped") - expect(bundle?.declarations).toStrictEqual([ - { - type: "input-variable", - name: "- value", - }, + expect(imported.bundles[0]?.id).toStrictEqual("keyInterpolateUnescaped") + expect(imported.bundles[0]?.declarations).toStrictEqual([ + { type: "input-variable", name: "- value" }, ]) - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ + expect(imported.variants[0]?.pattern).toStrictEqual([ { type: "text", value: "replace this " }, - { - type: "expression", - arg: { - type: "variable-reference", - name: "- value", - }, - }, + { type: "expression", arg: { type: "variable-reference", name: "- value" } }, ] satisfies Pattern) }) @@ -99,73 +88,62 @@ test("keyInterpolateWithFormatting", async () => { const imported = await runImportFiles({ keyInterpolateWithFormatting: "replace this {{value, format}}", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyInterpolateWithFormatting: "replace this {{value, format}}", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyInterpolateWithFormatting") - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ + expect(imported.bundles[0]?.id).toStrictEqual("keyInterpolateWithFormatting") + expect(imported.variants[0]?.pattern).toStrictEqual([ { type: "text", value: "replace this " }, { type: "expression", - arg: { - type: "variable-reference", - name: "value", - }, - annotation: { - type: "function-reference", - name: "format", - options: [], - }, + arg: { type: "variable-reference", name: "value" }, + annotation: { type: "function-reference", name: "format", options: [] }, }, ] satisfies Pattern) }) -test("keyContext", async () => { +test.todo("keyContext", async () => { const imported = await runImportFiles({ + // catch all + keyContext: "the variant", + // context: male keyContext_male: "the male variant", + // context: female keyContext_female: "the female variant", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ + keyContext: "the variant", keyContext_male: "the male variant", keyContext_female: "the female variant", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyContext") + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(3) - expect(bundle?.declarations).toStrictEqual([ - { - type: "input-variable", - name: "context", - }, + expect(imported.bundles[0]?.id).toStrictEqual("keyContext") + expect(imported.bundles[0]?.declarations).toStrictEqual([ + { type: "input-variable", name: "context" }, ]) - expect(bundle?.messages[0]?.selectors).toStrictEqual([ - { - type: "variable-reference", - name: "context", - }, + expect(imported?.messages[0]?.selectors).toStrictEqual([ + { type: "variable-reference", name: "context" }, ]) - expect(bundle?.messages[0]?.variants[0]).toStrictEqual( + expect(imported.variants[0]).toStrictEqual( expect.objectContaining({ - matches: [ - { - type: "literal-match", - key: "context", - value: "male", - }, - ], + matches: [{ type: "catchall-match", key: "context" }], + pattern: [{ type: "text", value: "the variant" }], + } satisfies Partial) + ) + expect(imported.variants[1]).toStrictEqual( + expect.objectContaining({ + matches: [{ type: "literal-match", key: "context", value: "male" }], pattern: [{ type: "text", value: "the male variant" }], } satisfies Partial) ) - expect(bundle?.messages[0]?.variants[1]).toStrictEqual( + expect(imported.variants[2]).toStrictEqual( expect.objectContaining({ - matches: [ - { - type: "literal-match", - key: "context", - value: "female", - }, - ], + matches: [{ type: "literal-match", key: "context", value: "female" }], pattern: [{ type: "text", value: "the female variant" }], } satisfies Partial) ) @@ -176,14 +154,14 @@ test("keyPluralSimple", async () => { keyPluralSimple_one: "the singular", keyPluralSimple_other: "the plural", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyPluralSimple_one: "the singular", keyPluralSimple_other: "the plural", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyPluralSimple") + expect(imported.bundles[0]?.id).toStrictEqual("keyPluralSimple") - expect(bundle?.declarations).toStrictEqual( + expect(imported.bundles[0]?.declarations).toStrictEqual( expect.arrayContaining([ { type: "input-variable", @@ -208,14 +186,14 @@ test("keyPluralSimple", async () => { ]) ) - expect(bundle?.messages[0]?.selectors).toStrictEqual([ + expect(imported?.messages[0]?.selectors).toStrictEqual([ { type: "variable-reference", name: "countPlural", }, ]) - expect(bundle?.messages[0]?.variants[0]).toStrictEqual( + expect(imported?.variants[0]).toStrictEqual( expect.objectContaining({ matches: [ { @@ -228,7 +206,7 @@ test("keyPluralSimple", async () => { } satisfies Partial) ) - expect(bundle?.messages[0]?.variants[1]).toStrictEqual( + expect(imported?.variants[1]).toStrictEqual( expect.objectContaining({ matches: [ { @@ -243,7 +221,7 @@ test("keyPluralSimple", async () => { }) test("keyPluralMultipleEgArabic", async () => { - const result = await runImportFiles({ + const imported = await runImportFiles({ keyPluralMultipleEgArabic_zero: "the plural form 0", keyPluralMultipleEgArabic_one: "the plural form 1", keyPluralMultipleEgArabic_two: "the plural form 2", @@ -251,7 +229,7 @@ test("keyPluralMultipleEgArabic", async () => { keyPluralMultipleEgArabic_many: "the plural form 4", keyPluralMultipleEgArabic_other: "the plural form 5", }) - expect(await runExportFiles(result)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyPluralMultipleEgArabic_zero: "the plural form 0", keyPluralMultipleEgArabic_one: "the plural form 1", keyPluralMultipleEgArabic_two: "the plural form 2", @@ -260,16 +238,13 @@ test("keyPluralMultipleEgArabic", async () => { keyPluralMultipleEgArabic_other: "the plural form 5", }) - const bundle = result.bundles.find((bundle) => bundle.id === "keyPluralMultipleEgArabic") + expect(imported.bundles[0]?.id).toStrictEqual("keyPluralMultipleEgArabic") - expect(bundle?.messages[0]?.selectors).toStrictEqual([ - { - type: "variable-reference", - name: "countPlural", - }, + expect(imported?.messages[0]?.selectors).toStrictEqual([ + { type: "variable-reference", name: "countPlural" }, ]) - expect(bundle?.declarations).toStrictEqual( + expect(imported.bundles[0]?.declarations).toStrictEqual( expect.arrayContaining([ { type: "input-variable", @@ -294,26 +269,25 @@ test("keyPluralMultipleEgArabic", async () => { ]) ) - const matches = bundle?.messages[0]?.variants.map( - (variant) => (variant.matches?.[0] as LiteralMatch).value - ) + const matches = imported.variants.map((variant) => (variant.matches?.[0] as LiteralMatch).value) + expect(matches).toStrictEqual(["zero", "one", "two", "few", "many", "other"]) - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ + expect(imported.variants[0]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 0" }, ]) - expect(bundle?.messages[0]?.variants[1]?.pattern).toStrictEqual([ + expect(imported.variants[1]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 1" }, ]) - expect(bundle?.messages[0]?.variants[2]?.pattern).toStrictEqual([ + expect(imported.variants[2]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 2" }, ]) - expect(bundle?.messages[0]?.variants[3]?.pattern).toStrictEqual([ + expect(imported.variants[3]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 3" }, ]) - expect(bundle?.messages[0]?.variants[4]?.pattern).toStrictEqual([ + expect(imported.variants[4]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 4" }, ]) - expect(bundle?.messages[0]?.variants[5]?.pattern).toStrictEqual([ + expect(imported.variants[5]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 5" }, ]) }) @@ -325,41 +299,41 @@ test("keyWithObjectValue", async () => { valueB: "more text", }, }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyWithObjectValue: { valueA: "return this with valueB", valueB: "more text", }, }) - const valueA = imported.bundles.find((bundle) => bundle.id === "keyWithObjectValue.valueA") - const valueB = imported.bundles.find((bundle) => bundle.id === "keyWithObjectValue.valueB") + expect(imported.bundles[0]?.id).toStrictEqual("keyWithObjectValue.valueA") + expect(imported.bundles[1]?.id).toStrictEqual("keyWithObjectValue.valueB") - expect(valueA?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "return this with valueB" }, - ] satisfies Pattern) - expect(valueB?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "more text" }, - ] satisfies Pattern) + expect( + imported.variants.find((v) => v.bundleId === "keyWithObjectValue.valueA")?.pattern + ).toStrictEqual([{ type: "text", value: "return this with valueB" }] satisfies Pattern) + expect( + imported.variants.find((v) => v.bundleId === "keyWithObjectValue.valueB")?.pattern + ).toStrictEqual([{ type: "text", value: "more text" }] satisfies Pattern) }) test("keyWithArrayValue", async () => { const imported = await runImportFiles({ keyWithArrayValue: ["multiple", "things"], }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyWithArrayValue: ["multiple", "things"], }) - const bundle1 = imported.bundles.find((bundle) => bundle.id === "keyWithArrayValue.0") - const bundle2 = imported.bundles.find((bundle) => bundle.id === "keyWithArrayValue.1") + expect(imported.bundles[0]?.id).toStrictEqual("keyWithArrayValue.0") + expect(imported.bundles[1]?.id).toStrictEqual("keyWithArrayValue.1") - expect(bundle1?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "multiple" }, - ] satisfies Pattern) - expect(bundle2?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "things" }, - ] satisfies Pattern) + expect( + imported.variants.find((v) => v.bundleId === "keyWithArrayValue.0")?.pattern + ).toStrictEqual([{ type: "text", value: "multiple" }] satisfies Pattern) + expect( + imported.variants.find((v) => v.bundleId === "keyWithArrayValue.1")?.pattern + ).toStrictEqual([{ type: "text", value: "things" }] satisfies Pattern) }) test("im- and exporting multiple files should succeed", async () => { @@ -383,10 +357,9 @@ test("im- and exporting multiple files should succeed", async () => { }, ], }) - const exported = await exportFiles({ - settings: {} as any, - bundles: imported.bundles as BundleNested[], - }) + + const exported = await runExportFiles(imported) + const exportedEn = JSON.parse( new TextDecoder().decode(exported.find((e) => e.locale === "en")?.content) ) @@ -429,10 +402,8 @@ test("it should handle namespaces", async () => { }, ], }) - const exported = await exportFiles({ - settings: {} as any, - bundles: imported.bundles as BundleNested[], - }) + const exported = await runExportFiles(imported) + const exportedCommon = JSON.parse( new TextDecoder().decode(exported.find((e) => e.name === "common-en.json")?.content) ) @@ -448,7 +419,7 @@ test("it should handle namespaces", async () => { }) }) -test("it should put new entities into the resource file without a namespace", async () => { +test("it should put new entities into the file without a namespace", async () => { const enNoNamespace = { blue_box: "value1", } @@ -474,30 +445,30 @@ test("it should put new entities into the resource file without a namespace", as ], }) - const newBundle: BundleNested = { - id: "happy_elephant", + const newBundle: Bundle = { + id: "new_bundle", declarations: [], - messages: [ - { - id: "happy_elephant_en", - bundleId: "happy_elephant", - locale: "en", - selectors: [], - variants: [ - { - id: "happy_elephant_en_*", - matches: [], - messageId: "happy_elephant_en", - pattern: [{ type: "text", value: "elephant" }], - }, - ], - }, - ], } - const exported = await exportFiles({ - settings: {} as any, - bundles: [...(imported.bundles as BundleNested[]), newBundle], + const newMessage: Message = { + id: "mock-29jas", + bundleId: "new_bundle", + locale: "en", + selectors: [], + } + + const newVariant: Variant = { + id: "mock-111sss", + matches: [], + messageId: "mock-29jas", + pattern: [{ type: "text", value: "elephant" }], + } + + const exported = await runExportFiles({ + bundles: [...imported.bundles, newBundle], + messages: [...imported.messages, newMessage], + //@ts-expect-error - variants are VariantImport which differs from the Variant type + variants: [...imported.variants, newVariant], }) const exportedNoNamespace = JSON.parse( @@ -510,7 +481,7 @@ test("it should put new entities into the resource file without a namespace", as expect(exportedNoNamespace).toStrictEqual({ blue_box: "value1", - happy_elephant: "elephant", + new_bundle: "elephant", }) expect(exportedCommon).toStrictEqual({ @@ -525,19 +496,21 @@ test("a key with a single variant should have no matches even if other keys are keyPluralSimple_other: "the plural", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ key: "value", keyPluralSimple_one: "the singular", keyPluralSimple_other: "the plural", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "key") + expect(imported.bundles).lengthOf(2) + expect(imported.messages).lengthOf(3) + expect(imported.variants).lengthOf(3) - expect(bundle?.messages[0]?.selectors).toStrictEqual([]) - expect(bundle?.messages[0]?.variants[0]?.matches).toStrictEqual([]) - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "value" }, - ]) + expect(imported.bundles[0]?.id).toStrictEqual("key") + + expect(imported.messages[0]?.selectors).toStrictEqual([]) + expect(imported.variants[0]?.matches).toStrictEqual([]) + expect(imported.variants[0]?.pattern).toStrictEqual([{ type: "text", value: "value" }]) }) // convenience wrapper for less testing code @@ -554,10 +527,35 @@ function runImportFiles(json: Record) { } // convenience wrapper for less testing code -async function runExportFiles(imported: Awaited>) { +async function runExportFiles(imported: Awaited>) { + // add ids which are undefined from the import + for (const message of imported.messages) { + if (message.id === undefined) { + message.id = `${Math.random() * 1000}` + } + } + for (const variant of imported.variants) { + if (variant.id === undefined) { + variant.id = `${Math.random() * 1000}` + } + if (variant.messageId === undefined) { + variant.messageId = imported.messages.find( + (m: any) => m.bundleId === variant.bundleId && m.locale === variant.locale + )?.id + } + } + const exported = await exportFiles({ settings: {} as any, - bundles: imported.bundles as BundleNested[], + bundles: imported.bundles, + messages: imported.messages as Message[], + variants: imported.variants as Variant[], }) + return exported +} + +// convenience wrapper for less testing code +async function runExportFilesParsed(imported: any) { + const exported = await runExportFiles(imported) return JSON.parse(new TextDecoder().decode(exported[0]?.content)) } diff --git a/inlang/source-code/sdk2/src/import-export/importFiles.test.ts b/inlang/source-code/sdk2/src/import-export/importFiles.test.ts new file mode 100644 index 0000000000..924ffa01bf --- /dev/null +++ b/inlang/source-code/sdk2/src/import-export/importFiles.test.ts @@ -0,0 +1,56 @@ +import { test, expect, vi } from "vitest"; +import { importFiles } from "./importFiles.js"; +import { loadProjectInMemory } from "../project/loadProjectInMemory.js"; +import { newProject } from "../project/newProject.js"; +import type { InlangPlugin } from "../plugin/schema.js"; + +test("it should insert a message as is if the id is provided", async () => { + const project = await loadProjectInMemory({ blob: await newProject() }); + + const mockPlugin: InlangPlugin = { + key: "mock", + importFiles: async () => ({ bundles: [], messages: [], variants: [] }), + }; + + const result = await importFiles({ + db: project.db, + files: [{ content: new Uint8Array(), locale: "mock" }], + pluginKey: "mock", + plugins: [mockPlugin], + settings: {} as any, + }); +}); + +test("it should match an existing message if the id is not provided", async () => {}); + +test("it should create a bundle for a message if the bundle does not exist to avoid foreign key conflicts", async () => { + const project = await loadProjectInMemory({ blob: await newProject() }); + + const mockPlugin: InlangPlugin = { + key: "mock", + importFiles: async () => ({ + bundles: [], + messages: [{ bundleId: "non-existent-bundle", locale: "en" }], + variants: [], + }), + }; + + await importFiles({ + db: project.db, + files: [{ content: new Uint8Array(), locale: "mock" }], + pluginKey: "mock", + plugins: [mockPlugin], + settings: {} as any, + }); + + const bundles = await project.db.selectFrom("bundle").selectAll().execute(); + + expect(bundles.length).toBe(1); + expect(bundles[0]?.id).toBe("non-existent-bundle"); +}); + +test("it should insert a variant as is if the id is provided", async () => {}); + +test("it should match an existing variant if the id is not provided", async () => {}); + +test("it should create a message for a variant if the message does not exist to avoid foreign key conflicts", async () => {}); diff --git a/inlang/source-code/sdk2/src/import-export/importFiles.ts b/inlang/source-code/sdk2/src/import-export/importFiles.ts new file mode 100644 index 0000000000..d46e720ecf --- /dev/null +++ b/inlang/source-code/sdk2/src/import-export/importFiles.ts @@ -0,0 +1,100 @@ +import type { Kysely } from "kysely"; +import { + PluginDoesNotImplementFunctionError, + PluginMissingError, +} from "../plugin/errors.js"; +import type { ProjectSettings } from "../json-schema/settings.js"; +import type { InlangDatabaseSchema, NewVariant } from "../database/schema.js"; +import type { InlangPlugin } from "../plugin/schema.js"; +import type { ImportFile } from "../project/api.js"; + +export async function importFiles(args: { + files: ImportFile[]; + readonly pluginKey: string; + readonly settings: ProjectSettings; + readonly plugins: readonly InlangPlugin[]; + readonly db: Kysely; +}) { + const plugin = args.plugins.find((p) => p.key === args.pluginKey); + + if (!plugin) throw new PluginMissingError({ plugin: args.pluginKey }); + + if (!plugin.importFiles) { + throw new PluginDoesNotImplementFunctionError({ + plugin: args.pluginKey, + function: "importFiles", + }); + } + + const imported = await plugin.importFiles({ + files: args.files, + settings: structuredClone(args.settings), + }); + + await args.db.transaction().execute(async (trx) => { + // upsert every bundle + for (const bundle of imported.bundles) { + await trx + .insertInto("bundle") + .values(bundle) + .onConflict((oc) => oc.column("id").doUpdateSet(bundle)) + .execute(); + } + // upsert every message + for (const message of imported.messages) { + // match the message by bundle id and locale if + // no id is provided by the importer + if (message.id === undefined) { + const exisingMessage = await trx + .selectFrom("message") + .where("bundleId", "=", message.bundleId) + .where("locale", "=", message.locale) + .select("id") + .executeTakeFirst(); + message.id = exisingMessage?.id; + } + await trx + .insertInto("message") + .values(message) + .onConflict((oc) => oc.column("id").doUpdateSet(message)) + .execute(); + } + // upsert every variant + for (const variant of imported.variants) { + // match the variant by message id and matches if + // no id is provided by the importer + if (variant.id === undefined) { + const existingMessage = await trx + .selectFrom("message") + .where("bundleId", "=", variant.bundleId) + .where("locale", "=", variant.locale) + .select("id") + .executeTakeFirstOrThrow(); + + const existingVariants = await trx + .selectFrom("variant") + .where("messageId", "=", existingMessage.id) + .selectAll() + .execute(); + + const existingVariant = existingVariants.find( + (v) => JSON.stringify(v.matches) === JSON.stringify(variant.matches) + ); + + variant.id = existingVariant?.id; + variant.messageId = existingMessage.id; + } + const toBeInsertedVariant: NewVariant = { + ...variant, + // @ts-expect-error - bundle id is provided by VariantImport but not needed when inserting + bundleId: undefined, + locale: undefined, + }; + await trx + .insertInto("variant") + .values(toBeInsertedVariant) + .onConflict((oc) => oc.column("id").doUpdateSet(toBeInsertedVariant)) + .execute(); + } + }); +} diff --git a/inlang/source-code/sdk2/src/import-export/index.ts b/inlang/source-code/sdk2/src/import-export/index.ts index 0a15ca1fe0..3bce1e7c57 100644 --- a/inlang/source-code/sdk2/src/import-export/index.ts +++ b/inlang/source-code/sdk2/src/import-export/index.ts @@ -5,38 +5,7 @@ import { } from "../plugin/errors.js"; import type { ProjectSettings } from "../json-schema/settings.js"; import type { InlangDatabaseSchema } from "../database/schema.js"; -import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; import type { InlangPlugin } from "../plugin/schema.js"; -import type { ImportFile } from "../project/api.js"; -import { upsertBundleNestedMatchByProperties } from "./upsertBundleNestedMatchByProperties.js"; - -export async function importFiles(opts: { - files: ImportFile[]; - readonly pluginKey: string; - readonly settings: ProjectSettings; - readonly plugins: readonly InlangPlugin[]; - readonly db: Kysely; -}) { - const plugin = opts.plugins.find((p) => p.key === opts.pluginKey); - if (!plugin) throw new PluginMissingError({ plugin: opts.pluginKey }); - if (!plugin.importFiles) { - throw new PluginDoesNotImplementFunctionError({ - plugin: opts.pluginKey, - function: "importFiles", - }); - } - - const { bundles } = await plugin.importFiles({ - files: opts.files, - settings: structuredClone(opts.settings), - }); - - const insertPromises = bundles.map((bundle) => - upsertBundleNestedMatchByProperties(opts.db, bundle) - ); - - await Promise.all(insertPromises); -} export async function exportFiles(opts: { readonly pluginKey: string; @@ -53,13 +22,15 @@ export async function exportFiles(opts: { }); } - const bundles = await selectBundleNested(opts.db) - .orderBy("id asc") - .selectAll() - .execute(); + const bundles = await opts.db.selectFrom("bundle").selectAll().execute(); + const messages = await opts.db.selectFrom("message").selectAll().execute(); + const variants = await opts.db.selectFrom("variant").selectAll().execute(); + const files = await plugin.exportFiles({ - bundles: bundles, settings: structuredClone(opts.settings), + bundles, + messages, + variants, }); return files; } diff --git a/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts b/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts index b5b685e141..3a1523c08a 100644 --- a/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts +++ b/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts @@ -1,13 +1,17 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { test, expect } from "vitest"; -import type { InlangPlugin } from "../plugin/schema.js"; -import type { BundleNested, Message } from "../database/schema.js"; +import type { + BundleImport, + InlangPlugin, + MessageImport, + VariantImport, +} from "../plugin/schema.js"; +import type { Message } from "../database/schema.js"; import type { Text } from "../json-schema/pattern.js"; -import { exportFiles, importFiles } from "./index.js"; +import { exportFiles } from "./index.js"; import { loadProjectInMemory } from "../project/loadProjectInMemory.js"; import { newProject } from "../project/newProject.js"; -import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; -import { insertBundleNested } from "../query-utilities/insertBundleNested.js"; +import { importFiles } from "./importFiles.js"; test("the file should be identical after a roundtrip if no modifications occured", async () => { const project = await loadProjectInMemory({ @@ -26,19 +30,25 @@ test("the file should be identical after a roundtrip if no modifications occured db: project.db, }); - const importedBundles = await selectBundleNested(project.db) + const importedBundles = await project.db + .selectFrom("bundle") + .selectAll() + .execute(); + const importedMessages = await project.db + .selectFrom("message") + .selectAll() + .execute(); + const importedVariants = await project.db + .selectFrom("variant") .selectAll() .execute(); - const importedMessages = importedBundles.flatMap((bundle) => bundle.messages); - const importedVariants = importedMessages.flatMap( - (message) => message.variants - ); expect(importedBundles.length).toBe(1); expect(importedMessages.length).toBe(1); expect(importedVariants.length).toBe(1); expect(importedBundles[0]?.id).toBe("hello_world"); expect(importedMessages[0]?.bundleId).toBe("hello_world"); + expect(importedVariants[0]?.messageId).toBe(importedMessages[0]?.id); const exportedFiles = await exportFiles({ pluginKey: "mock", @@ -57,44 +67,47 @@ test("a variant with an existing match should update the existing variant and no blob: await newProject(), }); - const bundleWithMatches: BundleNested = { - id: "mock-bundle-id", - declarations: [], + const existing = { + bundles: [ + { + id: "mock-bundle-id", + declarations: [], + }, + ], messages: [ { id: "mock-message-id", locale: "en", selectors: [], bundleId: "mock-bundle-id", - variants: [ + }, + ], + variants: [ + { + id: "mock-variant-id", + messageId: "mock-message-id", + matches: [ { - id: "mock-variant-id", - messageId: "mock-message-id", - matches: [ - { - type: "literal-match", - key: "color", - value: "blue", - }, - ], - pattern: [ - { - type: "text", - value: "You have blue eyes.", - }, - ], + type: "literal-match", + key: "color", + value: "blue", + }, + ], + pattern: [ + { + type: "text", + value: "You have blue eyes.", }, ], }, ], }; - const updatedBundleWithMatches = structuredClone(bundleWithMatches); + const updated = structuredClone(existing); // @ts-expect-error - we know this is a text pattern - updatedBundleWithMatches.messages[0].variants[0].pattern[0].value = - "You have beautiful blue eyes."; + updated.variants[0].pattern[0].value = "You have beautiful blue eyes."; - const enResource = JSON.stringify([updatedBundleWithMatches]); + const enResource = JSON.stringify(updated); await importFiles({ files: [{ content: new TextEncoder().encode(enResource), locale: "en" }], @@ -120,37 +133,31 @@ test("if a message for the bundle id and locale already exists, update it. don't blob: await newProject(), }); - const existingBundle: BundleNested = { - id: "mock-bundle-id", - declarations: [], + const existing: any = { + bundles: [{ id: "mock-bundle-id", declarations: [] }], messages: [ { id: "mock-message-id", locale: "en", - selectors: [ - { - type: "variable-reference", - name: "variable1", - }, - ], + selectors: [], bundleId: "mock-bundle-id", - variants: [], }, ], + variants: [], }; - const updatedBundle = structuredClone(existingBundle); - updatedBundle.messages[0]!.selectors = [ + const updated = structuredClone(existing); + updated.messages[0]!.selectors = [ { type: "variable-reference", name: "variable2", }, ] satisfies Message["selectors"]; - const enResource = JSON.stringify([updatedBundle]); + const enResource = new TextEncoder().encode(JSON.stringify(updated)); await importFiles({ - files: [{ content: new TextEncoder().encode(enResource), locale: "en" }], + files: [{ content: enResource, locale: "en" }], pluginKey: "mock", plugins: [mockPluginAst], settings: await project.settings.get(), @@ -190,14 +197,15 @@ test("keys should be ordered alphabetically for .json to minimize git diffs", as db: project.db, }); - await insertBundleNested( - project.db, - mockBundle({ - id: "c", - locale: "en", - text: "value3", - }) - ); + await project.db.insertInto("bundle").values({ id: "c" }).execute(); + await project.db + .insertInto("message") + .values({ id: "c-en", bundleId: "c", locale: "en" }) + .execute(); + await project.db + .insertInto("variant") + .values({ messageId: "c-en", pattern: [{ type: "text", value: "value3" }] }) + .execute(); const exportedFiles = await exportFiles({ pluginKey: "mock", @@ -218,21 +226,28 @@ test("keys should be ordered alphabetically for .json to minimize git diffs", as const mockPluginAst: InlangPlugin = { key: "mock", - exportFiles: async ({ bundles }) => { + exportFiles: async ({ bundles, messages, variants }) => { return [ { locale: "every", - name: "bundles.json", - content: new TextEncoder().encode(JSON.stringify(bundles)), + name: "x.json", + content: new TextEncoder().encode( + JSON.stringify({ bundles, messages, variants }) + ), }, ]; }, importFiles: async ({ files }) => { - return { - bundles: files.flatMap((file) => - JSON.parse(new TextDecoder().decode(file.content)) - ), - }; + let bundles: any[] = []; + let messages: any[] = []; + let variants: any[] = []; + for (const file of files) { + const parsed = JSON.parse(new TextDecoder().decode(file.content)); + bundles = [...bundles, ...parsed.bundles]; + messages = [...messages, ...parsed.messages]; + variants = [...variants, ...parsed.variants]; + } + return { bundles, messages, variants }; }, }; @@ -243,12 +258,13 @@ const mockPluginAst: InlangPlugin = { // increase maintainability const mockPluginSimple: InlangPlugin = { key: "mock", - exportFiles: async ({ bundles }) => { + exportFiles: async ({ messages, variants }) => { const jsons: any = {}; - const messages = bundles.flatMap((bundle) => bundle.messages); for (const message of messages) { const key = message.bundleId; - const value = (message.variants[0]?.pattern[0] as Text).value; + const value = ( + variants.find((v) => v.messageId === message.id)?.pattern[0] as Text + ).value; if (!jsons[message.locale]) { jsons[message.locale] = {}; } @@ -261,49 +277,38 @@ const mockPluginSimple: InlangPlugin = { })); }, importFiles: async ({ files }) => { - const bundles: BundleNested[] = []; + const bundles: BundleImport[] = []; + const messages: MessageImport[] = []; + const variants: VariantImport[] = []; for (const file of files) { const parsed = JSON.parse(new TextDecoder().decode(file.content)); for (const key in parsed) { - bundles.push( - mockBundle({ id: key, locale: file.locale, text: parsed[key] }) - ); + bundles.push({ + id: key, + declarations: [], + }); + messages.push({ + bundleId: key, + locale: file.locale, + selectors: [], + }); + variants.push({ + bundleId: key, + locale: file.locale, + matches: [], + pattern: [ + { + type: "text", + value: parsed[key], + }, + ], + }); } } return { bundles, + messages, + variants, }; }, }; - -function mockBundle(args: { - id: string; - locale: string; - text: string; -}): BundleNested { - return { - id: args.id, - declarations: [], - messages: [ - { - id: args.id + args.locale, - locale: args.locale, - selectors: [], - bundleId: args.id, - variants: [ - { - id: args.id + args.locale, - messageId: args.id + args.locale, - matches: [], - pattern: [ - { - type: "text", - value: args.text, - }, - ], - }, - ], - }, - ], - }; -} diff --git a/inlang/source-code/sdk2/src/plugin/errors.ts b/inlang/source-code/sdk2/src/plugin/errors.ts index 17bc9ca600..ba9b937b85 100644 --- a/inlang/source-code/sdk2/src/plugin/errors.ts +++ b/inlang/source-code/sdk2/src/plugin/errors.ts @@ -51,7 +51,7 @@ export class PluginSettingsAreInvalidError extends PluginError { export class PluginDoesNotImplementFunctionError extends PluginError { constructor(options: { plugin: string; function: string }) { super( - `The plugin "${options.plugin}" does not implement the "${options.function}" function`, + `The plugin "${options.plugin}" does not implement the "${options.function}" function.`, options ); this.name = "PluginDoesNotImplementFunction"; diff --git a/inlang/source-code/sdk2/src/plugin/schema.ts b/inlang/source-code/sdk2/src/plugin/schema.ts index 2a04481e5a..df6d08215e 100644 --- a/inlang/source-code/sdk2/src/plugin/schema.ts +++ b/inlang/source-code/sdk2/src/plugin/schema.ts @@ -1,7 +1,14 @@ import type { TObject } from "@sinclair/typebox"; import type { MessageV1 } from "../json-schema/old-v1-message/schemaV1.js"; import type { ProjectSettings } from "../json-schema/settings.js"; -import type { BundleNested, NewBundleNested } from "../database/schema.js"; +import type { + Bundle, + Message, + NewBundle, + NewMessage, + NewVariant, + Variant, +} from "../database/schema.js"; import type { ExportFile } from "../project/api.js"; export type InlangPlugin< @@ -52,10 +59,14 @@ export type InlangPlugin< }>; settings: ProjectSettings & ExternalSettings; // we expose the settings in case the importFunction needs to access the plugin config }) => MaybePromise<{ - bundles: NewBundleNested[]; + bundles: BundleImport[]; + messages: MessageImport[]; + variants: VariantImport[]; }>; exportFiles?: (args: { - bundles: BundleNested[]; + bundles: Bundle[]; + messages: Message[]; + variants: Variant[]; settings: ProjectSettings & ExternalSettings; }) => MaybePromise>; /** @@ -101,4 +112,48 @@ export type NodeFsPromisesSubset = { readdir: (path: string) => Promise; }; +/** + * A to be imported bundle. + */ +export type BundleImport = NewBundle; + +/** + * A to be imported message. + * + * The `id` property is omitted because it is generated by the SDK. + */ +export type MessageImport = Omit & { + /** + * If the id is not provided, the SDK will generate one. + */ + id?: string; +}; + +/** + * A to be imported variant. + * + * - The `id` and `messageId` properties are omitted because they are generated by the SDK. + * - The `bundleId` and `locale` properties are added to the import variant to match the variant + * with a message. + */ +export type VariantImport = Omit & { + /** + * If the id is not provided, the SDK will generate one. + */ + id?: string; + /** + * If the messageId is not provided, the SDK will match the variant + * with a message based on the `bundleId` and `locale` properties. + */ + messageId?: string; + /** + * Required to match the variant with a message in case the `id` and `messageId` are undefined. + */ + bundleId: string; + /** + * Required to match the variant with a message in case the `id` and `messageId` are undefined. + */ + locale: string; +}; + type MaybePromise = T | Promise; \ No newline at end of file diff --git a/inlang/source-code/sdk2/src/project/loadProject.ts b/inlang/source-code/sdk2/src/project/loadProject.ts index 672dd45da5..ef03f542b2 100644 --- a/inlang/source-code/sdk2/src/project/loadProject.ts +++ b/inlang/source-code/sdk2/src/project/loadProject.ts @@ -8,10 +8,11 @@ import { type PreprocessPluginBeforeImportFunction } from "../plugin/importPlugi import type { InlangProject } from "./api.js"; import { createProjectState } from "./state/state.js"; import { withLanguageTagToLocaleMigration } from "../migrations/v2/withLanguageTagToLocaleMigration.js"; -import { exportFiles, importFiles } from "../import-export/index.js"; +import { exportFiles } from "../import-export/index.js"; import { v4 } from "uuid"; import { initErrorReporting } from "../services/error-reporting/index.js"; import { maybeCaptureLoadedProject } from "./maybeCaptureTelemetry.js"; +import { importFiles } from "../import-export/importFiles.js"; /** * Common load project logic. diff --git a/inlang/source-code/sdk2/src/project/loadProjectFromDirectory.test.ts b/inlang/source-code/sdk2/src/project/loadProjectFromDirectory.test.ts index c2060540f8..413490b020 100644 --- a/inlang/source-code/sdk2/src/project/loadProjectFromDirectory.test.ts +++ b/inlang/source-code/sdk2/src/project/loadProjectFromDirectory.test.ts @@ -563,7 +563,7 @@ test("errors from importing translation files should be shown", async () => { const mockPlugin: InlangPlugin = { key: "mock-plugin", importFiles: async () => { - return { bundles: [] }; + return { bundles: [], messages: [], variants: [] }; }, toBeImportedFiles: async () => { return [{ path: "./some-file.json", locale: "mock" }]; @@ -597,7 +597,7 @@ test("errors from importing translation files that are ENOENT should not be show const mockPlugin: InlangPlugin = { key: "mock-plugin", importFiles: async () => { - return { bundles: [] }; + return { bundles: [], messages: [], variants: [] }; }, toBeImportedFiles: async () => { return [{ path: "./some-non-existing-file.json", locale: "mock" }]; @@ -640,7 +640,7 @@ test("it should pass toBeImportedMetadata on import", async () => { ]; }, importFiles: async () => { - return { bundles: [] }; + return { bundles: [], messages: [], variants: [] }; }, }; diff --git a/inlang/source-code/sdk2/src/project/saveProjectToDirectory.test.ts b/inlang/source-code/sdk2/src/project/saveProjectToDirectory.test.ts index 65d4c2e499..8b276d4ab5 100644 --- a/inlang/source-code/sdk2/src/project/saveProjectToDirectory.test.ts +++ b/inlang/source-code/sdk2/src/project/saveProjectToDirectory.test.ts @@ -4,8 +4,7 @@ import { Volume } from "memfs"; import { loadProjectInMemory } from "./loadProjectInMemory.js"; import { newProject } from "./newProject.js"; import type { InlangPlugin } from "../plugin/schema.js"; -import type { NewBundleNested } from "../database/schema.js"; -import { insertBundleNested } from "../query-utilities/insertBundleNested.js"; +import type { Bundle, NewMessage, Variant } from "../database/schema.js"; import { loadProjectFromDirectory } from "./loadProjectFromDirectory.js"; import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; import type { ProjectSettings } from "../json-schema/settings.js"; @@ -61,20 +60,12 @@ test("it should overwrite all files to the directory except the db.sqlite file", }); test("a roundtrip should work", async () => { - const mockBundleNested: NewBundleNested = { - id: "mock-bundle", - messages: [ - { - bundleId: "mock-bundle", - locale: "en", - selectors: [], - variants: [], - }, - ], - }; + const bundles: Bundle[] = [{ id: "mock-bundle", declarations: [] }]; + const messages: NewMessage[] = [{ bundleId: "mock-bundle", locale: "en" }]; + const variants: Variant[] = []; const volume = Volume.fromJSON({ - "/mock-file.json": JSON.stringify([mockBundleNested]), + "/mock-file.json": JSON.stringify({ bundles, messages, variants }), }); const mockPlugin: InlangPlugin = { @@ -83,8 +74,10 @@ test("a roundtrip should work", async () => { return [{ path: "/mock-file.json", locale: "mock" }]; }, importFiles: async ({ files }) => { - const bundles = JSON.parse(new TextDecoder().decode(files[0]?.content)); - return { bundles }; + const { bundles, messages, variants } = JSON.parse( + new TextDecoder().decode(files[0]?.content) + ); + return { bundles, messages, variants }; }, exportFiles: async ({ bundles }) => { return [ @@ -105,7 +98,8 @@ test("a roundtrip should work", async () => { providePlugins: [mockPlugin], }); - await insertBundleNested(project.db, mockBundleNested); + await project.db.insertInto("bundle").values(bundles).execute(); + await project.db.insertInto("message").values(messages).execute(); await saveProjectToDirectory({ fs: volume.promises as any, @@ -135,15 +129,27 @@ test("a roundtrip should work", async () => { expect(mockPlugin.importFiles).toHaveBeenCalled(); - const bundles = await selectBundleNested(project2.db).execute(); - - for (const bundle of bundles) { - for (const message of bundle.messages) { - delete (message as any).id; // Remove the dynamic id field - } - } - - expect(bundles).toEqual([expect.objectContaining(mockBundleNested)]); + const bundlesAfter = await project2.db + .selectFrom("bundle") + .selectAll() + .execute(); + const messagesAfter = await project2.db + .selectFrom("message") + .selectAll() + .execute(); + const variantsAfter = await project2.db + .selectFrom("variant") + .selectAll() + .execute(); + + expect(bundlesAfter).lengthOf(1); + expect(messagesAfter).lengthOf(1); + expect(variantsAfter).lengthOf(0); + + expect(bundlesAfter[0]).toStrictEqual(expect.objectContaining(bundles[0])); + expect(messagesAfter[0]).toStrictEqual( + expect.objectContaining(messagesAfter[0]) + ); }); test.todo( @@ -300,4 +306,4 @@ test("it should preserve the formatting of existing json resource files", async const fileAfterSave = await volume.promises.readFile("/foo/en.json", "utf-8"); expect(fileAfterSave).toBe(mockJson); -}); \ No newline at end of file +}); diff --git a/inlang/source-code/sdk2/src/project/saveProjectToDirectory.ts b/inlang/source-code/sdk2/src/project/saveProjectToDirectory.ts index bd92a9cc50..8e22a1592f 100644 --- a/inlang/source-code/sdk2/src/project/saveProjectToDirectory.ts +++ b/inlang/source-code/sdk2/src/project/saveProjectToDirectory.ts @@ -2,13 +2,13 @@ import type fs from "node:fs/promises"; import type { InlangProject } from "./api.js"; import path from "node:path"; -import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; import { toMessageV1 } from "../json-schema/old-v1-message/toMessageV1.js"; import { absolutePathFromProject, withAbsolutePaths, } from "./loadProjectFromDirectory.js"; import { detectJsonFormatting } from "../utilities/detectJsonFormatting.js"; +import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; export async function saveProjectToDirectory(args: { fs: typeof fs; @@ -36,13 +36,15 @@ export async function saveProjectToDirectory(args: { // run exporters const plugins = await args.project.plugins.get(); const settings = await args.project.settings.get(); - const bundles = await selectBundleNested(args.project.db).execute(); for (const plugin of plugins) { // old legacy remove with v3 if (plugin.saveMessages) { + // in-efficient re-qeuery but it's a legacy function that will be removed. + // the effort of adjusting the code to not re-query is not worth it. + const bundlesNested = await selectBundleNested(args.project.db).execute(); await plugin.saveMessages({ - messages: bundles.map((b) => toMessageV1(b)), + messages: bundlesNested.map((b) => toMessageV1(b)), // @ts-expect-error - legacy nodeishFs: withAbsolutePaths(args.fs, args.path), settings, @@ -50,8 +52,22 @@ export async function saveProjectToDirectory(args: { } if (plugin.exportFiles) { + const bundles = await args.project.db + .selectFrom("bundle") + .selectAll() + .execute(); + const messages = await args.project.db + .selectFrom("message") + .selectAll() + .execute(); + const variants = await args.project.db + .selectFrom("variant") + .selectAll() + .execute(); const files = await plugin.exportFiles({ bundles, + messages, + variants, settings, }); for (const file of files) { From 2bffc4d576010ff9af50c7c11620e4ae75c5ef16 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 19:23:08 -0400 Subject: [PATCH 02/11] refactor: dedicated export --- .../i18next/src/import-export/exportFiles.ts | 10 +- .../i18next/src/import-export/importFiles.ts | 70 ++-- .../src/import-export/roundtrip.test.ts | 344 +++++++++--------- .../src/import-export/importFiles.test.ts | 56 +++ .../sdk2/src/import-export/importFiles.ts | 100 +++++ .../sdk2/src/import-export/index.ts | 43 +-- .../sdk2/src/import-export/roundtrip.test.ts | 209 +++++------ inlang/source-code/sdk2/src/plugin/errors.ts | 2 +- inlang/source-code/sdk2/src/plugin/schema.ts | 61 +++- .../sdk2/src/project/loadProject.ts | 3 +- .../project/loadProjectFromDirectory.test.ts | 6 +- .../project/saveProjectToDirectory.test.ts | 60 +-- .../src/project/saveProjectToDirectory.ts | 22 +- 13 files changed, 586 insertions(+), 400 deletions(-) create mode 100644 inlang/source-code/sdk2/src/import-export/importFiles.test.ts create mode 100644 inlang/source-code/sdk2/src/import-export/importFiles.ts diff --git a/inlang/source-code/plugins/i18next/src/import-export/exportFiles.ts b/inlang/source-code/plugins/i18next/src/import-export/exportFiles.ts index 0db8bc9dc5..4a26ce6346 100644 --- a/inlang/source-code/plugins/i18next/src/import-export/exportFiles.ts +++ b/inlang/source-code/plugins/i18next/src/import-export/exportFiles.ts @@ -3,17 +3,19 @@ import type { Bundle, LiteralMatch, Message, Pattern, Variant } from "@inlang/sd import { type plugin } from "../plugin.js" import { unflatten } from "flat" -export const exportFiles: NonNullable<(typeof plugin)["exportFiles"]> = async ({ bundles }) => { +export const exportFiles: NonNullable<(typeof plugin)["exportFiles"]> = async ({ + bundles, + messages, + variants, +}) => { const result: Record> = {} const resultNamespaces: Record>> = {} - const messages = bundles.flatMap((bundle) => bundle.messages) - for (const message of messages) { const serializedMessages = serializeMessage( bundles.find((b) => b.id === message.bundleId)!, message, - message.variants + variants.filter((v) => v.messageId === message.id) ) for (const message of serializedMessages) { diff --git a/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts b/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts index 8dc11ac435..09bd82275b 100644 --- a/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts +++ b/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts @@ -1,20 +1,17 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ -import type { - Bundle, - BundleNested, - Message, - Pattern, - VariableReference, - Variant, -} from "@inlang/sdk2" +import type { Pattern, VariableReference, Variant } from "@inlang/sdk2" import { type plugin } from "../plugin.js" import { flatten } from "flat" +import type { + BundleImport, + MessageImport, + VariantImport, +} from "../../../../sdk2/dist/plugin/schema.js" export const importFiles: NonNullable<(typeof plugin)["importFiles"]> = async ({ files }) => { - const result: BundleNested[] = [] - const bundles: Bundle[] = [] - const messages: Message[] = [] - const variants: Variant[] = [] + const bundles: BundleImport[] = [] + const messages: MessageImport[] = [] + const variants: VariantImport[] = [] for (const file of files) { const namespace = file.toBeImportedFilesMetadata?.namespace @@ -26,41 +23,27 @@ export const importFiles: NonNullable<(typeof plugin)["importFiles"]> = async ({ // merge the bundle declarations const uniqueBundleIds = [...new Set(bundles.map((bundle) => bundle.id))] - const uniqueBundles: Bundle[] = uniqueBundleIds.map((id) => { + const uniqueBundles: BundleImport[] = uniqueBundleIds.map((id) => { const _bundles = bundles.filter((bundle) => bundle.id === id) const declarations = removeDuplicates(_bundles.flatMap((bundle) => bundle.declarations)) return { id, declarations } }) - // establishing nesting - for (const bundle of uniqueBundles) { - const bundleNested: BundleNested = { ...bundle, messages: [] } - - // @ts-expect-error - casting the type here - bundleNested.messages = messages.filter((message) => message.bundleId === bundle.id) - - for (const message of bundleNested.messages) { - message.variants = variants.filter((variant) => variant.messageId === message.id) - } - - result.push(bundleNested) - } - - return { bundles: result } + return { bundles: uniqueBundles, messages, variants } } function parseFile(args: { namespace?: string; locale: string; content: ArrayBuffer }): { - bundles: Bundle[] - messages: Message[] - variants: Variant[] + bundles: BundleImport[] + messages: MessageImport[] + variants: VariantImport[] } { const resource: Record = flatten( JSON.parse(new TextDecoder().decode(args.content)) ) - const bundles: Bundle[] = [] - const messages: Message[] = [] - const variants: Variant[] = [] + const bundles: BundleImport[] = [] + const messages: MessageImport[] = [] + const variants: VariantImport[] = [] for (const key in resource) { const value = resource[key]! @@ -84,7 +67,7 @@ function parseMessage(args: { value: string locale: string resource: Record -}): { bundle: Bundle; message: Message; variant: Variant } { +}): { bundle: BundleImport; message: MessageImport; variant: VariantImport } { const pattern = parsePattern(args.value) // i18next suffixes keys with context or plurals @@ -96,7 +79,7 @@ function parseMessage(args: { bundleId = `${args.namespace}:${bundleId}` } - const bundle: Bundle = { + const bundle: BundleImport = { id: bundleId, declarations: pattern.variableReferences.map((variableReference) => ({ type: "input-variable", @@ -104,16 +87,15 @@ function parseMessage(args: { })), } - const message: Message = { - id: "", + const message: MessageImport = { bundleId: bundleId, selectors: [], locale: args.locale, } - const variant: Variant = { - id: "", - messageId: "", + const variant: VariantImport = { + bundleId: bundleId, + locale: args.locale, matches: [], pattern: pattern.result, } @@ -259,12 +241,6 @@ function parseMessage(args: { bundle.declarations = removeDuplicates(bundle.declarations) - // i18next suffixes keys with context or plurals - // "friend_female_one" -> "friend" - message.id = `${bundle.id};;locale=${args.locale};;` - variant.id = `${message.id};;match=${variant.matches.join(",")};;` - variant.messageId = message.id - return { bundle, message, variant } } diff --git a/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts b/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts index ae234506a5..1f0f2cac04 100644 --- a/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts +++ b/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts @@ -1,70 +1,68 @@ import { expect, test } from "vitest" import { importFiles } from "./importFiles.js" -import type { BundleNested, LiteralMatch, Pattern, Variant } from "@inlang/sdk2" +import { + type Bundle, + type LiteralMatch, + type Message, + type Pattern, + type Variant, +} from "@inlang/sdk2" import { exportFiles } from "./exportFiles.js" test("single key value", async () => { const imported = await runImportFiles({ key: "value", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ key: "value", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "key") + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(1) - expect(bundle?.messages[0]?.selectors).toStrictEqual([]) - expect(bundle?.messages[0]?.variants[0]?.matches).toStrictEqual([]) - - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "value" }, - ]) + expect(imported.bundles[0]?.id).toStrictEqual("key") + expect(imported.bundles[0]?.declarations).toStrictEqual([]) + expect(imported.messages[0]?.selectors).toStrictEqual([]) + expect(imported.variants[0]?.matches).toStrictEqual([]) + expect(imported.variants[0]?.pattern).toStrictEqual([{ type: "text", value: "value" }]) }) test("key deep", async () => { const imported = await runImportFiles({ - keyDeep: { - inner: "value", - }, + keyDeep: { inner: "value" }, }) - expect(await runExportFiles(imported)).toStrictEqual({ - keyDeep: { - inner: "value", - }, + expect(await runExportFilesParsed(imported)).toStrictEqual({ + keyDeep: { inner: "value" }, }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyDeep.inner") - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "value" }, - ]) + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(1) + + expect(imported.bundles[0]?.id).toStrictEqual("keyDeep.inner") + expect(imported.variants[0]?.pattern).toStrictEqual([{ type: "text", value: "value" }]) }) test("keyInterpolate", async () => { const imported = await runImportFiles({ keyInterpolate: "replace this {{value}}", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyInterpolate: "replace this {{value}}", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyInterpolate") + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(1) - expect(bundle?.declarations).toStrictEqual([ - { - type: "input-variable", - name: "value", - }, + expect(imported.bundles[0]?.declarations).toStrictEqual([ + { type: "input-variable", name: "value" }, ]) - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ + expect(imported.variants[0]?.pattern).toStrictEqual([ { type: "text", value: "replace this " }, - { - type: "expression", - arg: { - type: "variable-reference", - name: "value", - }, - }, + { type: "expression", arg: { type: "variable-reference", name: "value" } }, ] satisfies Pattern) }) @@ -72,26 +70,17 @@ test("keyInterpolateUnescaped", async () => { const imported = await runImportFiles({ keyInterpolateUnescaped: "replace this {{- value}}", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyInterpolateUnescaped: "replace this {{- value}}", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyInterpolateUnescaped") - expect(bundle?.declarations).toStrictEqual([ - { - type: "input-variable", - name: "- value", - }, + expect(imported.bundles[0]?.id).toStrictEqual("keyInterpolateUnescaped") + expect(imported.bundles[0]?.declarations).toStrictEqual([ + { type: "input-variable", name: "- value" }, ]) - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ + expect(imported.variants[0]?.pattern).toStrictEqual([ { type: "text", value: "replace this " }, - { - type: "expression", - arg: { - type: "variable-reference", - name: "- value", - }, - }, + { type: "expression", arg: { type: "variable-reference", name: "- value" } }, ] satisfies Pattern) }) @@ -99,73 +88,62 @@ test("keyInterpolateWithFormatting", async () => { const imported = await runImportFiles({ keyInterpolateWithFormatting: "replace this {{value, format}}", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyInterpolateWithFormatting: "replace this {{value, format}}", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyInterpolateWithFormatting") - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ + expect(imported.bundles[0]?.id).toStrictEqual("keyInterpolateWithFormatting") + expect(imported.variants[0]?.pattern).toStrictEqual([ { type: "text", value: "replace this " }, { type: "expression", - arg: { - type: "variable-reference", - name: "value", - }, - annotation: { - type: "function-reference", - name: "format", - options: [], - }, + arg: { type: "variable-reference", name: "value" }, + annotation: { type: "function-reference", name: "format", options: [] }, }, ] satisfies Pattern) }) -test("keyContext", async () => { +test.todo("keyContext", async () => { const imported = await runImportFiles({ + // catch all + keyContext: "the variant", + // context: male keyContext_male: "the male variant", + // context: female keyContext_female: "the female variant", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ + keyContext: "the variant", keyContext_male: "the male variant", keyContext_female: "the female variant", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyContext") + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(3) - expect(bundle?.declarations).toStrictEqual([ - { - type: "input-variable", - name: "context", - }, + expect(imported.bundles[0]?.id).toStrictEqual("keyContext") + expect(imported.bundles[0]?.declarations).toStrictEqual([ + { type: "input-variable", name: "context" }, ]) - expect(bundle?.messages[0]?.selectors).toStrictEqual([ - { - type: "variable-reference", - name: "context", - }, + expect(imported?.messages[0]?.selectors).toStrictEqual([ + { type: "variable-reference", name: "context" }, ]) - expect(bundle?.messages[0]?.variants[0]).toStrictEqual( + expect(imported.variants[0]).toStrictEqual( expect.objectContaining({ - matches: [ - { - type: "literal-match", - key: "context", - value: "male", - }, - ], + matches: [{ type: "catchall-match", key: "context" }], + pattern: [{ type: "text", value: "the variant" }], + } satisfies Partial) + ) + expect(imported.variants[1]).toStrictEqual( + expect.objectContaining({ + matches: [{ type: "literal-match", key: "context", value: "male" }], pattern: [{ type: "text", value: "the male variant" }], } satisfies Partial) ) - expect(bundle?.messages[0]?.variants[1]).toStrictEqual( + expect(imported.variants[2]).toStrictEqual( expect.objectContaining({ - matches: [ - { - type: "literal-match", - key: "context", - value: "female", - }, - ], + matches: [{ type: "literal-match", key: "context", value: "female" }], pattern: [{ type: "text", value: "the female variant" }], } satisfies Partial) ) @@ -176,14 +154,14 @@ test("keyPluralSimple", async () => { keyPluralSimple_one: "the singular", keyPluralSimple_other: "the plural", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyPluralSimple_one: "the singular", keyPluralSimple_other: "the plural", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "keyPluralSimple") + expect(imported.bundles[0]?.id).toStrictEqual("keyPluralSimple") - expect(bundle?.declarations).toStrictEqual( + expect(imported.bundles[0]?.declarations).toStrictEqual( expect.arrayContaining([ { type: "input-variable", @@ -208,14 +186,14 @@ test("keyPluralSimple", async () => { ]) ) - expect(bundle?.messages[0]?.selectors).toStrictEqual([ + expect(imported?.messages[0]?.selectors).toStrictEqual([ { type: "variable-reference", name: "countPlural", }, ]) - expect(bundle?.messages[0]?.variants[0]).toStrictEqual( + expect(imported?.variants[0]).toStrictEqual( expect.objectContaining({ matches: [ { @@ -228,7 +206,7 @@ test("keyPluralSimple", async () => { } satisfies Partial) ) - expect(bundle?.messages[0]?.variants[1]).toStrictEqual( + expect(imported?.variants[1]).toStrictEqual( expect.objectContaining({ matches: [ { @@ -243,7 +221,7 @@ test("keyPluralSimple", async () => { }) test("keyPluralMultipleEgArabic", async () => { - const result = await runImportFiles({ + const imported = await runImportFiles({ keyPluralMultipleEgArabic_zero: "the plural form 0", keyPluralMultipleEgArabic_one: "the plural form 1", keyPluralMultipleEgArabic_two: "the plural form 2", @@ -251,7 +229,7 @@ test("keyPluralMultipleEgArabic", async () => { keyPluralMultipleEgArabic_many: "the plural form 4", keyPluralMultipleEgArabic_other: "the plural form 5", }) - expect(await runExportFiles(result)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyPluralMultipleEgArabic_zero: "the plural form 0", keyPluralMultipleEgArabic_one: "the plural form 1", keyPluralMultipleEgArabic_two: "the plural form 2", @@ -260,16 +238,13 @@ test("keyPluralMultipleEgArabic", async () => { keyPluralMultipleEgArabic_other: "the plural form 5", }) - const bundle = result.bundles.find((bundle) => bundle.id === "keyPluralMultipleEgArabic") + expect(imported.bundles[0]?.id).toStrictEqual("keyPluralMultipleEgArabic") - expect(bundle?.messages[0]?.selectors).toStrictEqual([ - { - type: "variable-reference", - name: "countPlural", - }, + expect(imported?.messages[0]?.selectors).toStrictEqual([ + { type: "variable-reference", name: "countPlural" }, ]) - expect(bundle?.declarations).toStrictEqual( + expect(imported.bundles[0]?.declarations).toStrictEqual( expect.arrayContaining([ { type: "input-variable", @@ -294,26 +269,25 @@ test("keyPluralMultipleEgArabic", async () => { ]) ) - const matches = bundle?.messages[0]?.variants.map( - (variant) => (variant.matches?.[0] as LiteralMatch).value - ) + const matches = imported.variants.map((variant) => (variant.matches?.[0] as LiteralMatch).value) + expect(matches).toStrictEqual(["zero", "one", "two", "few", "many", "other"]) - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ + expect(imported.variants[0]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 0" }, ]) - expect(bundle?.messages[0]?.variants[1]?.pattern).toStrictEqual([ + expect(imported.variants[1]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 1" }, ]) - expect(bundle?.messages[0]?.variants[2]?.pattern).toStrictEqual([ + expect(imported.variants[2]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 2" }, ]) - expect(bundle?.messages[0]?.variants[3]?.pattern).toStrictEqual([ + expect(imported.variants[3]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 3" }, ]) - expect(bundle?.messages[0]?.variants[4]?.pattern).toStrictEqual([ + expect(imported.variants[4]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 4" }, ]) - expect(bundle?.messages[0]?.variants[5]?.pattern).toStrictEqual([ + expect(imported.variants[5]?.pattern).toStrictEqual([ { type: "text", value: "the plural form 5" }, ]) }) @@ -325,41 +299,41 @@ test("keyWithObjectValue", async () => { valueB: "more text", }, }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyWithObjectValue: { valueA: "return this with valueB", valueB: "more text", }, }) - const valueA = imported.bundles.find((bundle) => bundle.id === "keyWithObjectValue.valueA") - const valueB = imported.bundles.find((bundle) => bundle.id === "keyWithObjectValue.valueB") + expect(imported.bundles[0]?.id).toStrictEqual("keyWithObjectValue.valueA") + expect(imported.bundles[1]?.id).toStrictEqual("keyWithObjectValue.valueB") - expect(valueA?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "return this with valueB" }, - ] satisfies Pattern) - expect(valueB?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "more text" }, - ] satisfies Pattern) + expect( + imported.variants.find((v) => v.bundleId === "keyWithObjectValue.valueA")?.pattern + ).toStrictEqual([{ type: "text", value: "return this with valueB" }] satisfies Pattern) + expect( + imported.variants.find((v) => v.bundleId === "keyWithObjectValue.valueB")?.pattern + ).toStrictEqual([{ type: "text", value: "more text" }] satisfies Pattern) }) test("keyWithArrayValue", async () => { const imported = await runImportFiles({ keyWithArrayValue: ["multiple", "things"], }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ keyWithArrayValue: ["multiple", "things"], }) - const bundle1 = imported.bundles.find((bundle) => bundle.id === "keyWithArrayValue.0") - const bundle2 = imported.bundles.find((bundle) => bundle.id === "keyWithArrayValue.1") + expect(imported.bundles[0]?.id).toStrictEqual("keyWithArrayValue.0") + expect(imported.bundles[1]?.id).toStrictEqual("keyWithArrayValue.1") - expect(bundle1?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "multiple" }, - ] satisfies Pattern) - expect(bundle2?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "things" }, - ] satisfies Pattern) + expect( + imported.variants.find((v) => v.bundleId === "keyWithArrayValue.0")?.pattern + ).toStrictEqual([{ type: "text", value: "multiple" }] satisfies Pattern) + expect( + imported.variants.find((v) => v.bundleId === "keyWithArrayValue.1")?.pattern + ).toStrictEqual([{ type: "text", value: "things" }] satisfies Pattern) }) test("im- and exporting multiple files should succeed", async () => { @@ -383,10 +357,9 @@ test("im- and exporting multiple files should succeed", async () => { }, ], }) - const exported = await exportFiles({ - settings: {} as any, - bundles: imported.bundles as BundleNested[], - }) + + const exported = await runExportFiles(imported) + const exportedEn = JSON.parse( new TextDecoder().decode(exported.find((e) => e.locale === "en")?.content) ) @@ -429,10 +402,8 @@ test("it should handle namespaces", async () => { }, ], }) - const exported = await exportFiles({ - settings: {} as any, - bundles: imported.bundles as BundleNested[], - }) + const exported = await runExportFiles(imported) + const exportedCommon = JSON.parse( new TextDecoder().decode(exported.find((e) => e.name === "common-en.json")?.content) ) @@ -448,7 +419,7 @@ test("it should handle namespaces", async () => { }) }) -test("it should put new entities into the resource file without a namespace", async () => { +test("it should put new entities into the file without a namespace", async () => { const enNoNamespace = { blue_box: "value1", } @@ -474,30 +445,30 @@ test("it should put new entities into the resource file without a namespace", as ], }) - const newBundle: BundleNested = { - id: "happy_elephant", + const newBundle: Bundle = { + id: "new_bundle", declarations: [], - messages: [ - { - id: "happy_elephant_en", - bundleId: "happy_elephant", - locale: "en", - selectors: [], - variants: [ - { - id: "happy_elephant_en_*", - matches: [], - messageId: "happy_elephant_en", - pattern: [{ type: "text", value: "elephant" }], - }, - ], - }, - ], } - const exported = await exportFiles({ - settings: {} as any, - bundles: [...(imported.bundles as BundleNested[]), newBundle], + const newMessage: Message = { + id: "mock-29jas", + bundleId: "new_bundle", + locale: "en", + selectors: [], + } + + const newVariant: Variant = { + id: "mock-111sss", + matches: [], + messageId: "mock-29jas", + pattern: [{ type: "text", value: "elephant" }], + } + + const exported = await runExportFiles({ + bundles: [...imported.bundles, newBundle], + messages: [...imported.messages, newMessage], + //@ts-expect-error - variants are VariantImport which differs from the Variant type + variants: [...imported.variants, newVariant], }) const exportedNoNamespace = JSON.parse( @@ -510,7 +481,7 @@ test("it should put new entities into the resource file without a namespace", as expect(exportedNoNamespace).toStrictEqual({ blue_box: "value1", - happy_elephant: "elephant", + new_bundle: "elephant", }) expect(exportedCommon).toStrictEqual({ @@ -525,19 +496,21 @@ test("a key with a single variant should have no matches even if other keys are keyPluralSimple_other: "the plural", }) - expect(await runExportFiles(imported)).toStrictEqual({ + expect(await runExportFilesParsed(imported)).toStrictEqual({ key: "value", keyPluralSimple_one: "the singular", keyPluralSimple_other: "the plural", }) - const bundle = imported.bundles.find((bundle) => bundle.id === "key") + expect(imported.bundles).lengthOf(2) + expect(imported.messages).lengthOf(3) + expect(imported.variants).lengthOf(3) - expect(bundle?.messages[0]?.selectors).toStrictEqual([]) - expect(bundle?.messages[0]?.variants[0]?.matches).toStrictEqual([]) - expect(bundle?.messages[0]?.variants[0]?.pattern).toStrictEqual([ - { type: "text", value: "value" }, - ]) + expect(imported.bundles[0]?.id).toStrictEqual("key") + + expect(imported.messages[0]?.selectors).toStrictEqual([]) + expect(imported.variants[0]?.matches).toStrictEqual([]) + expect(imported.variants[0]?.pattern).toStrictEqual([{ type: "text", value: "value" }]) }) // convenience wrapper for less testing code @@ -554,10 +527,35 @@ function runImportFiles(json: Record) { } // convenience wrapper for less testing code -async function runExportFiles(imported: Awaited>) { +async function runExportFiles(imported: Awaited>) { + // add ids which are undefined from the import + for (const message of imported.messages) { + if (message.id === undefined) { + message.id = `${Math.random() * 1000}` + } + } + for (const variant of imported.variants) { + if (variant.id === undefined) { + variant.id = `${Math.random() * 1000}` + } + if (variant.messageId === undefined) { + variant.messageId = imported.messages.find( + (m: any) => m.bundleId === variant.bundleId && m.locale === variant.locale + )?.id + } + } + const exported = await exportFiles({ settings: {} as any, - bundles: imported.bundles as BundleNested[], + bundles: imported.bundles, + messages: imported.messages as Message[], + variants: imported.variants as Variant[], }) + return exported +} + +// convenience wrapper for less testing code +async function runExportFilesParsed(imported: any) { + const exported = await runExportFiles(imported) return JSON.parse(new TextDecoder().decode(exported[0]?.content)) } diff --git a/inlang/source-code/sdk2/src/import-export/importFiles.test.ts b/inlang/source-code/sdk2/src/import-export/importFiles.test.ts new file mode 100644 index 0000000000..924ffa01bf --- /dev/null +++ b/inlang/source-code/sdk2/src/import-export/importFiles.test.ts @@ -0,0 +1,56 @@ +import { test, expect, vi } from "vitest"; +import { importFiles } from "./importFiles.js"; +import { loadProjectInMemory } from "../project/loadProjectInMemory.js"; +import { newProject } from "../project/newProject.js"; +import type { InlangPlugin } from "../plugin/schema.js"; + +test("it should insert a message as is if the id is provided", async () => { + const project = await loadProjectInMemory({ blob: await newProject() }); + + const mockPlugin: InlangPlugin = { + key: "mock", + importFiles: async () => ({ bundles: [], messages: [], variants: [] }), + }; + + const result = await importFiles({ + db: project.db, + files: [{ content: new Uint8Array(), locale: "mock" }], + pluginKey: "mock", + plugins: [mockPlugin], + settings: {} as any, + }); +}); + +test("it should match an existing message if the id is not provided", async () => {}); + +test("it should create a bundle for a message if the bundle does not exist to avoid foreign key conflicts", async () => { + const project = await loadProjectInMemory({ blob: await newProject() }); + + const mockPlugin: InlangPlugin = { + key: "mock", + importFiles: async () => ({ + bundles: [], + messages: [{ bundleId: "non-existent-bundle", locale: "en" }], + variants: [], + }), + }; + + await importFiles({ + db: project.db, + files: [{ content: new Uint8Array(), locale: "mock" }], + pluginKey: "mock", + plugins: [mockPlugin], + settings: {} as any, + }); + + const bundles = await project.db.selectFrom("bundle").selectAll().execute(); + + expect(bundles.length).toBe(1); + expect(bundles[0]?.id).toBe("non-existent-bundle"); +}); + +test("it should insert a variant as is if the id is provided", async () => {}); + +test("it should match an existing variant if the id is not provided", async () => {}); + +test("it should create a message for a variant if the message does not exist to avoid foreign key conflicts", async () => {}); diff --git a/inlang/source-code/sdk2/src/import-export/importFiles.ts b/inlang/source-code/sdk2/src/import-export/importFiles.ts new file mode 100644 index 0000000000..d46e720ecf --- /dev/null +++ b/inlang/source-code/sdk2/src/import-export/importFiles.ts @@ -0,0 +1,100 @@ +import type { Kysely } from "kysely"; +import { + PluginDoesNotImplementFunctionError, + PluginMissingError, +} from "../plugin/errors.js"; +import type { ProjectSettings } from "../json-schema/settings.js"; +import type { InlangDatabaseSchema, NewVariant } from "../database/schema.js"; +import type { InlangPlugin } from "../plugin/schema.js"; +import type { ImportFile } from "../project/api.js"; + +export async function importFiles(args: { + files: ImportFile[]; + readonly pluginKey: string; + readonly settings: ProjectSettings; + readonly plugins: readonly InlangPlugin[]; + readonly db: Kysely; +}) { + const plugin = args.plugins.find((p) => p.key === args.pluginKey); + + if (!plugin) throw new PluginMissingError({ plugin: args.pluginKey }); + + if (!plugin.importFiles) { + throw new PluginDoesNotImplementFunctionError({ + plugin: args.pluginKey, + function: "importFiles", + }); + } + + const imported = await plugin.importFiles({ + files: args.files, + settings: structuredClone(args.settings), + }); + + await args.db.transaction().execute(async (trx) => { + // upsert every bundle + for (const bundle of imported.bundles) { + await trx + .insertInto("bundle") + .values(bundle) + .onConflict((oc) => oc.column("id").doUpdateSet(bundle)) + .execute(); + } + // upsert every message + for (const message of imported.messages) { + // match the message by bundle id and locale if + // no id is provided by the importer + if (message.id === undefined) { + const exisingMessage = await trx + .selectFrom("message") + .where("bundleId", "=", message.bundleId) + .where("locale", "=", message.locale) + .select("id") + .executeTakeFirst(); + message.id = exisingMessage?.id; + } + await trx + .insertInto("message") + .values(message) + .onConflict((oc) => oc.column("id").doUpdateSet(message)) + .execute(); + } + // upsert every variant + for (const variant of imported.variants) { + // match the variant by message id and matches if + // no id is provided by the importer + if (variant.id === undefined) { + const existingMessage = await trx + .selectFrom("message") + .where("bundleId", "=", variant.bundleId) + .where("locale", "=", variant.locale) + .select("id") + .executeTakeFirstOrThrow(); + + const existingVariants = await trx + .selectFrom("variant") + .where("messageId", "=", existingMessage.id) + .selectAll() + .execute(); + + const existingVariant = existingVariants.find( + (v) => JSON.stringify(v.matches) === JSON.stringify(variant.matches) + ); + + variant.id = existingVariant?.id; + variant.messageId = existingMessage.id; + } + const toBeInsertedVariant: NewVariant = { + ...variant, + // @ts-expect-error - bundle id is provided by VariantImport but not needed when inserting + bundleId: undefined, + locale: undefined, + }; + await trx + .insertInto("variant") + .values(toBeInsertedVariant) + .onConflict((oc) => oc.column("id").doUpdateSet(toBeInsertedVariant)) + .execute(); + } + }); +} diff --git a/inlang/source-code/sdk2/src/import-export/index.ts b/inlang/source-code/sdk2/src/import-export/index.ts index 0a15ca1fe0..3bce1e7c57 100644 --- a/inlang/source-code/sdk2/src/import-export/index.ts +++ b/inlang/source-code/sdk2/src/import-export/index.ts @@ -5,38 +5,7 @@ import { } from "../plugin/errors.js"; import type { ProjectSettings } from "../json-schema/settings.js"; import type { InlangDatabaseSchema } from "../database/schema.js"; -import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; import type { InlangPlugin } from "../plugin/schema.js"; -import type { ImportFile } from "../project/api.js"; -import { upsertBundleNestedMatchByProperties } from "./upsertBundleNestedMatchByProperties.js"; - -export async function importFiles(opts: { - files: ImportFile[]; - readonly pluginKey: string; - readonly settings: ProjectSettings; - readonly plugins: readonly InlangPlugin[]; - readonly db: Kysely; -}) { - const plugin = opts.plugins.find((p) => p.key === opts.pluginKey); - if (!plugin) throw new PluginMissingError({ plugin: opts.pluginKey }); - if (!plugin.importFiles) { - throw new PluginDoesNotImplementFunctionError({ - plugin: opts.pluginKey, - function: "importFiles", - }); - } - - const { bundles } = await plugin.importFiles({ - files: opts.files, - settings: structuredClone(opts.settings), - }); - - const insertPromises = bundles.map((bundle) => - upsertBundleNestedMatchByProperties(opts.db, bundle) - ); - - await Promise.all(insertPromises); -} export async function exportFiles(opts: { readonly pluginKey: string; @@ -53,13 +22,15 @@ export async function exportFiles(opts: { }); } - const bundles = await selectBundleNested(opts.db) - .orderBy("id asc") - .selectAll() - .execute(); + const bundles = await opts.db.selectFrom("bundle").selectAll().execute(); + const messages = await opts.db.selectFrom("message").selectAll().execute(); + const variants = await opts.db.selectFrom("variant").selectAll().execute(); + const files = await plugin.exportFiles({ - bundles: bundles, settings: structuredClone(opts.settings), + bundles, + messages, + variants, }); return files; } diff --git a/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts b/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts index b5b685e141..3a1523c08a 100644 --- a/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts +++ b/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts @@ -1,13 +1,17 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { test, expect } from "vitest"; -import type { InlangPlugin } from "../plugin/schema.js"; -import type { BundleNested, Message } from "../database/schema.js"; +import type { + BundleImport, + InlangPlugin, + MessageImport, + VariantImport, +} from "../plugin/schema.js"; +import type { Message } from "../database/schema.js"; import type { Text } from "../json-schema/pattern.js"; -import { exportFiles, importFiles } from "./index.js"; +import { exportFiles } from "./index.js"; import { loadProjectInMemory } from "../project/loadProjectInMemory.js"; import { newProject } from "../project/newProject.js"; -import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; -import { insertBundleNested } from "../query-utilities/insertBundleNested.js"; +import { importFiles } from "./importFiles.js"; test("the file should be identical after a roundtrip if no modifications occured", async () => { const project = await loadProjectInMemory({ @@ -26,19 +30,25 @@ test("the file should be identical after a roundtrip if no modifications occured db: project.db, }); - const importedBundles = await selectBundleNested(project.db) + const importedBundles = await project.db + .selectFrom("bundle") + .selectAll() + .execute(); + const importedMessages = await project.db + .selectFrom("message") + .selectAll() + .execute(); + const importedVariants = await project.db + .selectFrom("variant") .selectAll() .execute(); - const importedMessages = importedBundles.flatMap((bundle) => bundle.messages); - const importedVariants = importedMessages.flatMap( - (message) => message.variants - ); expect(importedBundles.length).toBe(1); expect(importedMessages.length).toBe(1); expect(importedVariants.length).toBe(1); expect(importedBundles[0]?.id).toBe("hello_world"); expect(importedMessages[0]?.bundleId).toBe("hello_world"); + expect(importedVariants[0]?.messageId).toBe(importedMessages[0]?.id); const exportedFiles = await exportFiles({ pluginKey: "mock", @@ -57,44 +67,47 @@ test("a variant with an existing match should update the existing variant and no blob: await newProject(), }); - const bundleWithMatches: BundleNested = { - id: "mock-bundle-id", - declarations: [], + const existing = { + bundles: [ + { + id: "mock-bundle-id", + declarations: [], + }, + ], messages: [ { id: "mock-message-id", locale: "en", selectors: [], bundleId: "mock-bundle-id", - variants: [ + }, + ], + variants: [ + { + id: "mock-variant-id", + messageId: "mock-message-id", + matches: [ { - id: "mock-variant-id", - messageId: "mock-message-id", - matches: [ - { - type: "literal-match", - key: "color", - value: "blue", - }, - ], - pattern: [ - { - type: "text", - value: "You have blue eyes.", - }, - ], + type: "literal-match", + key: "color", + value: "blue", + }, + ], + pattern: [ + { + type: "text", + value: "You have blue eyes.", }, ], }, ], }; - const updatedBundleWithMatches = structuredClone(bundleWithMatches); + const updated = structuredClone(existing); // @ts-expect-error - we know this is a text pattern - updatedBundleWithMatches.messages[0].variants[0].pattern[0].value = - "You have beautiful blue eyes."; + updated.variants[0].pattern[0].value = "You have beautiful blue eyes."; - const enResource = JSON.stringify([updatedBundleWithMatches]); + const enResource = JSON.stringify(updated); await importFiles({ files: [{ content: new TextEncoder().encode(enResource), locale: "en" }], @@ -120,37 +133,31 @@ test("if a message for the bundle id and locale already exists, update it. don't blob: await newProject(), }); - const existingBundle: BundleNested = { - id: "mock-bundle-id", - declarations: [], + const existing: any = { + bundles: [{ id: "mock-bundle-id", declarations: [] }], messages: [ { id: "mock-message-id", locale: "en", - selectors: [ - { - type: "variable-reference", - name: "variable1", - }, - ], + selectors: [], bundleId: "mock-bundle-id", - variants: [], }, ], + variants: [], }; - const updatedBundle = structuredClone(existingBundle); - updatedBundle.messages[0]!.selectors = [ + const updated = structuredClone(existing); + updated.messages[0]!.selectors = [ { type: "variable-reference", name: "variable2", }, ] satisfies Message["selectors"]; - const enResource = JSON.stringify([updatedBundle]); + const enResource = new TextEncoder().encode(JSON.stringify(updated)); await importFiles({ - files: [{ content: new TextEncoder().encode(enResource), locale: "en" }], + files: [{ content: enResource, locale: "en" }], pluginKey: "mock", plugins: [mockPluginAst], settings: await project.settings.get(), @@ -190,14 +197,15 @@ test("keys should be ordered alphabetically for .json to minimize git diffs", as db: project.db, }); - await insertBundleNested( - project.db, - mockBundle({ - id: "c", - locale: "en", - text: "value3", - }) - ); + await project.db.insertInto("bundle").values({ id: "c" }).execute(); + await project.db + .insertInto("message") + .values({ id: "c-en", bundleId: "c", locale: "en" }) + .execute(); + await project.db + .insertInto("variant") + .values({ messageId: "c-en", pattern: [{ type: "text", value: "value3" }] }) + .execute(); const exportedFiles = await exportFiles({ pluginKey: "mock", @@ -218,21 +226,28 @@ test("keys should be ordered alphabetically for .json to minimize git diffs", as const mockPluginAst: InlangPlugin = { key: "mock", - exportFiles: async ({ bundles }) => { + exportFiles: async ({ bundles, messages, variants }) => { return [ { locale: "every", - name: "bundles.json", - content: new TextEncoder().encode(JSON.stringify(bundles)), + name: "x.json", + content: new TextEncoder().encode( + JSON.stringify({ bundles, messages, variants }) + ), }, ]; }, importFiles: async ({ files }) => { - return { - bundles: files.flatMap((file) => - JSON.parse(new TextDecoder().decode(file.content)) - ), - }; + let bundles: any[] = []; + let messages: any[] = []; + let variants: any[] = []; + for (const file of files) { + const parsed = JSON.parse(new TextDecoder().decode(file.content)); + bundles = [...bundles, ...parsed.bundles]; + messages = [...messages, ...parsed.messages]; + variants = [...variants, ...parsed.variants]; + } + return { bundles, messages, variants }; }, }; @@ -243,12 +258,13 @@ const mockPluginAst: InlangPlugin = { // increase maintainability const mockPluginSimple: InlangPlugin = { key: "mock", - exportFiles: async ({ bundles }) => { + exportFiles: async ({ messages, variants }) => { const jsons: any = {}; - const messages = bundles.flatMap((bundle) => bundle.messages); for (const message of messages) { const key = message.bundleId; - const value = (message.variants[0]?.pattern[0] as Text).value; + const value = ( + variants.find((v) => v.messageId === message.id)?.pattern[0] as Text + ).value; if (!jsons[message.locale]) { jsons[message.locale] = {}; } @@ -261,49 +277,38 @@ const mockPluginSimple: InlangPlugin = { })); }, importFiles: async ({ files }) => { - const bundles: BundleNested[] = []; + const bundles: BundleImport[] = []; + const messages: MessageImport[] = []; + const variants: VariantImport[] = []; for (const file of files) { const parsed = JSON.parse(new TextDecoder().decode(file.content)); for (const key in parsed) { - bundles.push( - mockBundle({ id: key, locale: file.locale, text: parsed[key] }) - ); + bundles.push({ + id: key, + declarations: [], + }); + messages.push({ + bundleId: key, + locale: file.locale, + selectors: [], + }); + variants.push({ + bundleId: key, + locale: file.locale, + matches: [], + pattern: [ + { + type: "text", + value: parsed[key], + }, + ], + }); } } return { bundles, + messages, + variants, }; }, }; - -function mockBundle(args: { - id: string; - locale: string; - text: string; -}): BundleNested { - return { - id: args.id, - declarations: [], - messages: [ - { - id: args.id + args.locale, - locale: args.locale, - selectors: [], - bundleId: args.id, - variants: [ - { - id: args.id + args.locale, - messageId: args.id + args.locale, - matches: [], - pattern: [ - { - type: "text", - value: args.text, - }, - ], - }, - ], - }, - ], - }; -} diff --git a/inlang/source-code/sdk2/src/plugin/errors.ts b/inlang/source-code/sdk2/src/plugin/errors.ts index 17bc9ca600..ba9b937b85 100644 --- a/inlang/source-code/sdk2/src/plugin/errors.ts +++ b/inlang/source-code/sdk2/src/plugin/errors.ts @@ -51,7 +51,7 @@ export class PluginSettingsAreInvalidError extends PluginError { export class PluginDoesNotImplementFunctionError extends PluginError { constructor(options: { plugin: string; function: string }) { super( - `The plugin "${options.plugin}" does not implement the "${options.function}" function`, + `The plugin "${options.plugin}" does not implement the "${options.function}" function.`, options ); this.name = "PluginDoesNotImplementFunction"; diff --git a/inlang/source-code/sdk2/src/plugin/schema.ts b/inlang/source-code/sdk2/src/plugin/schema.ts index 2a04481e5a..df6d08215e 100644 --- a/inlang/source-code/sdk2/src/plugin/schema.ts +++ b/inlang/source-code/sdk2/src/plugin/schema.ts @@ -1,7 +1,14 @@ import type { TObject } from "@sinclair/typebox"; import type { MessageV1 } from "../json-schema/old-v1-message/schemaV1.js"; import type { ProjectSettings } from "../json-schema/settings.js"; -import type { BundleNested, NewBundleNested } from "../database/schema.js"; +import type { + Bundle, + Message, + NewBundle, + NewMessage, + NewVariant, + Variant, +} from "../database/schema.js"; import type { ExportFile } from "../project/api.js"; export type InlangPlugin< @@ -52,10 +59,14 @@ export type InlangPlugin< }>; settings: ProjectSettings & ExternalSettings; // we expose the settings in case the importFunction needs to access the plugin config }) => MaybePromise<{ - bundles: NewBundleNested[]; + bundles: BundleImport[]; + messages: MessageImport[]; + variants: VariantImport[]; }>; exportFiles?: (args: { - bundles: BundleNested[]; + bundles: Bundle[]; + messages: Message[]; + variants: Variant[]; settings: ProjectSettings & ExternalSettings; }) => MaybePromise>; /** @@ -101,4 +112,48 @@ export type NodeFsPromisesSubset = { readdir: (path: string) => Promise; }; +/** + * A to be imported bundle. + */ +export type BundleImport = NewBundle; + +/** + * A to be imported message. + * + * The `id` property is omitted because it is generated by the SDK. + */ +export type MessageImport = Omit & { + /** + * If the id is not provided, the SDK will generate one. + */ + id?: string; +}; + +/** + * A to be imported variant. + * + * - The `id` and `messageId` properties are omitted because they are generated by the SDK. + * - The `bundleId` and `locale` properties are added to the import variant to match the variant + * with a message. + */ +export type VariantImport = Omit & { + /** + * If the id is not provided, the SDK will generate one. + */ + id?: string; + /** + * If the messageId is not provided, the SDK will match the variant + * with a message based on the `bundleId` and `locale` properties. + */ + messageId?: string; + /** + * Required to match the variant with a message in case the `id` and `messageId` are undefined. + */ + bundleId: string; + /** + * Required to match the variant with a message in case the `id` and `messageId` are undefined. + */ + locale: string; +}; + type MaybePromise = T | Promise; \ No newline at end of file diff --git a/inlang/source-code/sdk2/src/project/loadProject.ts b/inlang/source-code/sdk2/src/project/loadProject.ts index 672dd45da5..ef03f542b2 100644 --- a/inlang/source-code/sdk2/src/project/loadProject.ts +++ b/inlang/source-code/sdk2/src/project/loadProject.ts @@ -8,10 +8,11 @@ import { type PreprocessPluginBeforeImportFunction } from "../plugin/importPlugi import type { InlangProject } from "./api.js"; import { createProjectState } from "./state/state.js"; import { withLanguageTagToLocaleMigration } from "../migrations/v2/withLanguageTagToLocaleMigration.js"; -import { exportFiles, importFiles } from "../import-export/index.js"; +import { exportFiles } from "../import-export/index.js"; import { v4 } from "uuid"; import { initErrorReporting } from "../services/error-reporting/index.js"; import { maybeCaptureLoadedProject } from "./maybeCaptureTelemetry.js"; +import { importFiles } from "../import-export/importFiles.js"; /** * Common load project logic. diff --git a/inlang/source-code/sdk2/src/project/loadProjectFromDirectory.test.ts b/inlang/source-code/sdk2/src/project/loadProjectFromDirectory.test.ts index c2060540f8..413490b020 100644 --- a/inlang/source-code/sdk2/src/project/loadProjectFromDirectory.test.ts +++ b/inlang/source-code/sdk2/src/project/loadProjectFromDirectory.test.ts @@ -563,7 +563,7 @@ test("errors from importing translation files should be shown", async () => { const mockPlugin: InlangPlugin = { key: "mock-plugin", importFiles: async () => { - return { bundles: [] }; + return { bundles: [], messages: [], variants: [] }; }, toBeImportedFiles: async () => { return [{ path: "./some-file.json", locale: "mock" }]; @@ -597,7 +597,7 @@ test("errors from importing translation files that are ENOENT should not be show const mockPlugin: InlangPlugin = { key: "mock-plugin", importFiles: async () => { - return { bundles: [] }; + return { bundles: [], messages: [], variants: [] }; }, toBeImportedFiles: async () => { return [{ path: "./some-non-existing-file.json", locale: "mock" }]; @@ -640,7 +640,7 @@ test("it should pass toBeImportedMetadata on import", async () => { ]; }, importFiles: async () => { - return { bundles: [] }; + return { bundles: [], messages: [], variants: [] }; }, }; diff --git a/inlang/source-code/sdk2/src/project/saveProjectToDirectory.test.ts b/inlang/source-code/sdk2/src/project/saveProjectToDirectory.test.ts index 65d4c2e499..8b276d4ab5 100644 --- a/inlang/source-code/sdk2/src/project/saveProjectToDirectory.test.ts +++ b/inlang/source-code/sdk2/src/project/saveProjectToDirectory.test.ts @@ -4,8 +4,7 @@ import { Volume } from "memfs"; import { loadProjectInMemory } from "./loadProjectInMemory.js"; import { newProject } from "./newProject.js"; import type { InlangPlugin } from "../plugin/schema.js"; -import type { NewBundleNested } from "../database/schema.js"; -import { insertBundleNested } from "../query-utilities/insertBundleNested.js"; +import type { Bundle, NewMessage, Variant } from "../database/schema.js"; import { loadProjectFromDirectory } from "./loadProjectFromDirectory.js"; import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; import type { ProjectSettings } from "../json-schema/settings.js"; @@ -61,20 +60,12 @@ test("it should overwrite all files to the directory except the db.sqlite file", }); test("a roundtrip should work", async () => { - const mockBundleNested: NewBundleNested = { - id: "mock-bundle", - messages: [ - { - bundleId: "mock-bundle", - locale: "en", - selectors: [], - variants: [], - }, - ], - }; + const bundles: Bundle[] = [{ id: "mock-bundle", declarations: [] }]; + const messages: NewMessage[] = [{ bundleId: "mock-bundle", locale: "en" }]; + const variants: Variant[] = []; const volume = Volume.fromJSON({ - "/mock-file.json": JSON.stringify([mockBundleNested]), + "/mock-file.json": JSON.stringify({ bundles, messages, variants }), }); const mockPlugin: InlangPlugin = { @@ -83,8 +74,10 @@ test("a roundtrip should work", async () => { return [{ path: "/mock-file.json", locale: "mock" }]; }, importFiles: async ({ files }) => { - const bundles = JSON.parse(new TextDecoder().decode(files[0]?.content)); - return { bundles }; + const { bundles, messages, variants } = JSON.parse( + new TextDecoder().decode(files[0]?.content) + ); + return { bundles, messages, variants }; }, exportFiles: async ({ bundles }) => { return [ @@ -105,7 +98,8 @@ test("a roundtrip should work", async () => { providePlugins: [mockPlugin], }); - await insertBundleNested(project.db, mockBundleNested); + await project.db.insertInto("bundle").values(bundles).execute(); + await project.db.insertInto("message").values(messages).execute(); await saveProjectToDirectory({ fs: volume.promises as any, @@ -135,15 +129,27 @@ test("a roundtrip should work", async () => { expect(mockPlugin.importFiles).toHaveBeenCalled(); - const bundles = await selectBundleNested(project2.db).execute(); - - for (const bundle of bundles) { - for (const message of bundle.messages) { - delete (message as any).id; // Remove the dynamic id field - } - } - - expect(bundles).toEqual([expect.objectContaining(mockBundleNested)]); + const bundlesAfter = await project2.db + .selectFrom("bundle") + .selectAll() + .execute(); + const messagesAfter = await project2.db + .selectFrom("message") + .selectAll() + .execute(); + const variantsAfter = await project2.db + .selectFrom("variant") + .selectAll() + .execute(); + + expect(bundlesAfter).lengthOf(1); + expect(messagesAfter).lengthOf(1); + expect(variantsAfter).lengthOf(0); + + expect(bundlesAfter[0]).toStrictEqual(expect.objectContaining(bundles[0])); + expect(messagesAfter[0]).toStrictEqual( + expect.objectContaining(messagesAfter[0]) + ); }); test.todo( @@ -300,4 +306,4 @@ test("it should preserve the formatting of existing json resource files", async const fileAfterSave = await volume.promises.readFile("/foo/en.json", "utf-8"); expect(fileAfterSave).toBe(mockJson); -}); \ No newline at end of file +}); diff --git a/inlang/source-code/sdk2/src/project/saveProjectToDirectory.ts b/inlang/source-code/sdk2/src/project/saveProjectToDirectory.ts index bd92a9cc50..8e22a1592f 100644 --- a/inlang/source-code/sdk2/src/project/saveProjectToDirectory.ts +++ b/inlang/source-code/sdk2/src/project/saveProjectToDirectory.ts @@ -2,13 +2,13 @@ import type fs from "node:fs/promises"; import type { InlangProject } from "./api.js"; import path from "node:path"; -import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; import { toMessageV1 } from "../json-schema/old-v1-message/toMessageV1.js"; import { absolutePathFromProject, withAbsolutePaths, } from "./loadProjectFromDirectory.js"; import { detectJsonFormatting } from "../utilities/detectJsonFormatting.js"; +import { selectBundleNested } from "../query-utilities/selectBundleNested.js"; export async function saveProjectToDirectory(args: { fs: typeof fs; @@ -36,13 +36,15 @@ export async function saveProjectToDirectory(args: { // run exporters const plugins = await args.project.plugins.get(); const settings = await args.project.settings.get(); - const bundles = await selectBundleNested(args.project.db).execute(); for (const plugin of plugins) { // old legacy remove with v3 if (plugin.saveMessages) { + // in-efficient re-qeuery but it's a legacy function that will be removed. + // the effort of adjusting the code to not re-query is not worth it. + const bundlesNested = await selectBundleNested(args.project.db).execute(); await plugin.saveMessages({ - messages: bundles.map((b) => toMessageV1(b)), + messages: bundlesNested.map((b) => toMessageV1(b)), // @ts-expect-error - legacy nodeishFs: withAbsolutePaths(args.fs, args.path), settings, @@ -50,8 +52,22 @@ export async function saveProjectToDirectory(args: { } if (plugin.exportFiles) { + const bundles = await args.project.db + .selectFrom("bundle") + .selectAll() + .execute(); + const messages = await args.project.db + .selectFrom("message") + .selectAll() + .execute(); + const variants = await args.project.db + .selectFrom("variant") + .selectAll() + .execute(); const files = await plugin.exportFiles({ bundles, + messages, + variants, settings, }); for (const file of files) { From dde3b7474b650a98d2af7cda4872c5d773109851 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 19:27:32 -0400 Subject: [PATCH 03/11] expose sqlite3 error required for try catch clauses --- lix/packages/sqlite-wasm-kysely/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lix/packages/sqlite-wasm-kysely/src/index.ts b/lix/packages/sqlite-wasm-kysely/src/index.ts index 3719ce6f33..629d38d9bb 100644 --- a/lix/packages/sqlite-wasm-kysely/src/index.ts +++ b/lix/packages/sqlite-wasm-kysely/src/index.ts @@ -1,2 +1,3 @@ +export { SQLite3Error } from "@eliaspourquoi/sqlite-node-wasm"; export * from "./util/index.js"; export { createDialect } from "./dialect.js"; From 0b8a14515907504df0239ed59711754ce9157bf4 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:17:11 -0400 Subject: [PATCH 04/11] improve: types --- inlang/source-code/sdk2/src/plugin/schema.ts | 58 +++++++++++++------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/inlang/source-code/sdk2/src/plugin/schema.ts b/inlang/source-code/sdk2/src/plugin/schema.ts index df6d08215e..5dbb051bf6 100644 --- a/inlang/source-code/sdk2/src/plugin/schema.ts +++ b/inlang/source-code/sdk2/src/plugin/schema.ts @@ -136,24 +136,44 @@ export type MessageImport = Omit & { * - The `bundleId` and `locale` properties are added to the import variant to match the variant * with a message. */ -export type VariantImport = Omit & { - /** - * If the id is not provided, the SDK will generate one. - */ - id?: string; - /** - * If the messageId is not provided, the SDK will match the variant - * with a message based on the `bundleId` and `locale` properties. - */ - messageId?: string; - /** - * Required to match the variant with a message in case the `id` and `messageId` are undefined. - */ - bundleId: string; - /** - * Required to match the variant with a message in case the `id` and `messageId` are undefined. - */ - locale: string; -}; +export type VariantImport = + | (NewVariant & { + /** + * If the id is not provided, the SDK will generate one. + */ + id: string; + /** + * If the messageId is not provided, the SDK will match the variant + * with a message based on the `messageBundleId` and `messageLocale` properties. + */ + messageId: string; + /** + * Required to match the variant with a message in case the `id` and `messageId` are undefined. + */ + messageBundleId?: undefined; + /** + * Required to match the variant with a message in case the `id` and `messageId` are undefined. + */ + messageLocale?: undefined; + }) + | (Omit & { + /** + * If the id is not provided, the SDK will generate one. + */ + id?: undefined; + /** + * If the messageId is not provided, the SDK will match the variant + * with a message based on the `messageBundleId` and `messageLocale` properties. + */ + messageId?: undefined; + /** + * Required to match the variant with a message in case the `id` and `messageId` are undefined. + */ + messageBundleId: string; + /** + * Required to match the variant with a message in case the `id` and `messageId` are undefined. + */ + messageLocale: string; + }); type MaybePromise = T | Promise; \ No newline at end of file From e497bf671a7c046696284fe73090ef935f27696c Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:19:02 -0400 Subject: [PATCH 05/11] add: upserts --- .../src/import-export/importFiles.test.ts | 171 +++++++++++++++++- .../sdk2/src/import-export/importFiles.ts | 71 ++++++-- .../sdk2/src/import-export/roundtrip.test.ts | 4 +- 3 files changed, 222 insertions(+), 24 deletions(-) diff --git a/inlang/source-code/sdk2/src/import-export/importFiles.test.ts b/inlang/source-code/sdk2/src/import-export/importFiles.test.ts index 924ffa01bf..0cd4636f94 100644 --- a/inlang/source-code/sdk2/src/import-export/importFiles.test.ts +++ b/inlang/source-code/sdk2/src/import-export/importFiles.test.ts @@ -1,4 +1,4 @@ -import { test, expect, vi } from "vitest"; +import { test, expect } from "vitest"; import { importFiles } from "./importFiles.js"; import { loadProjectInMemory } from "../project/loadProjectInMemory.js"; import { newProject } from "../project/newProject.js"; @@ -9,21 +9,74 @@ test("it should insert a message as is if the id is provided", async () => { const mockPlugin: InlangPlugin = { key: "mock", - importFiles: async () => ({ bundles: [], messages: [], variants: [] }), + importFiles: async () => ({ + bundles: [{ id: "mock-bundle" }], + messages: [{ id: "alfa23", bundleId: "mock-bundle", locale: "en" }], + variants: [], + }), }; - const result = await importFiles({ + await importFiles({ db: project.db, files: [{ content: new Uint8Array(), locale: "mock" }], pluginKey: "mock", plugins: [mockPlugin], settings: {} as any, }); + + const messages = await project.db.selectFrom("message").selectAll().execute(); + + expect(messages.length).toBe(1); + expect(messages[0]?.id).toBe("alfa23"); }); -test("it should match an existing message if the id is not provided", async () => {}); +test("it should match an existing message if the id is not provided", async () => { + const project = await loadProjectInMemory({ blob: await newProject() }); + + await project.db.insertInto("bundle").values({ id: "mock-bundle" }).execute(); + await project.db + .insertInto("message") + .values({ + id: "alfa23", + bundleId: "mock-bundle", + locale: "en", + selectors: [], + }) + .execute(); -test("it should create a bundle for a message if the bundle does not exist to avoid foreign key conflicts", async () => { + const mockPlugin: InlangPlugin = { + key: "mock", + importFiles: async () => ({ + bundles: [], + messages: [ + { + bundleId: "mock-bundle", + locale: "en", + selectors: [{ type: "variable-reference", name: "platform" }], + }, + ], + variants: [], + }), + }; + + await importFiles({ + db: project.db, + files: [{ content: new Uint8Array(), locale: "mock" }], + pluginKey: "mock", + plugins: [mockPlugin], + settings: {} as any, + }); + + const messages = await project.db.selectFrom("message").selectAll().execute(); + + expect(messages.length).toBe(1); + expect(messages[0]?.id).toBe("alfa23"); + expect(messages[0]?.selectors).toStrictEqual([ + { type: "variable-reference", name: "platform" }, + ]); +}); + +test("it should create a bundle for a message if the bundle does not exist to avoid foreign key conflicts and enable partial imports", async () => { const project = await loadProjectInMemory({ blob: await newProject() }); const mockPlugin: InlangPlugin = { @@ -49,8 +102,110 @@ test("it should create a bundle for a message if the bundle does not exist to av expect(bundles[0]?.id).toBe("non-existent-bundle"); }); -test("it should insert a variant as is if the id is provided", async () => {}); +test("it should insert a variant as is if the id is provided", async () => { + const project = await loadProjectInMemory({ blob: await newProject() }); + + await project.db.insertInto("bundle").values({ id: "mock-bundle" }).execute(); + await project.db + .insertInto("message") + .values({ + id: "mock-message", + bundleId: "mock-bundle", + locale: "en", + }) + .execute(); + + const mockPlugin: InlangPlugin = { + key: "mock", + importFiles: async () => ({ + bundles: [], + messages: [], + variants: [{ id: "variant-id-23", messageId: "mock-message" }], + }), + }; + + await importFiles({ + db: project.db, + files: [{ content: new Uint8Array(), locale: "mock" }], + pluginKey: "mock", + plugins: [mockPlugin], + settings: {} as any, + }); + + const variants = await project.db.selectFrom("variant").selectAll().execute(); + + expect(variants.length).toBe(1); + expect(variants[0]?.id).toBe("variant-id-23"); +}); + +test("it should match an existing variant if the id is not provided", async () => { + const project = await loadProjectInMemory({ blob: await newProject() }); + + await project.db.insertInto("bundle").values({ id: "mock-bundle" }).execute(); + await project.db + .insertInto("message") + .values({ + id: "mock-message", + bundleId: "mock-bundle", + locale: "en", + }) + .execute(); -test("it should match an existing variant if the id is not provided", async () => {}); + const mockPlugin: InlangPlugin = { + key: "mock", + importFiles: async () => ({ + bundles: [], + messages: [], + variants: [{ messageBundleId: "mock-bundle", messageLocale: "en" }], + }), + }; -test("it should create a message for a variant if the message does not exist to avoid foreign key conflicts", async () => {}); + await importFiles({ + db: project.db, + files: [{ content: new Uint8Array(), locale: "mock" }], + pluginKey: "mock", + plugins: [mockPlugin], + settings: {} as any, + }); + + const variants = await project.db.selectFrom("variant").selectAll().execute(); + + expect(variants.length).toBe(1); + expect(variants[0]?.messageId).toBe("mock-message"); + expect(variants[0]?.id).toBeDefined(); +}); + +test("it should create a message for a variant if the message does not exist to avoid foreign key conflicts and enable partial imports", async () => { + const project = await loadProjectInMemory({ blob: await newProject() }); + + await project.db.insertInto("bundle").values({ id: "mock-bundle" }).execute(); + + const mockPlugin: InlangPlugin = { + key: "mock", + importFiles: async () => ({ + bundles: [], + messages: [], + variants: [{ messageBundleId: "mock-bundle", messageLocale: "en" }], + }), + }; + + await importFiles({ + db: project.db, + files: [{ content: new Uint8Array(), locale: "mock" }], + pluginKey: "mock", + plugins: [mockPlugin], + settings: {} as any, + }); + + const bundles = await project.db.selectFrom("bundle").selectAll().execute(); + const messages = await project.db.selectFrom("message").selectAll().execute(); + const variants = await project.db.selectFrom("variant").selectAll().execute(); + + expect(bundles.length).toBe(1); + expect(messages.length).toBe(1); + expect(variants.length).toBe(1); + + expect(messages[0]?.bundleId).toBe("mock-bundle"); + expect(messages[0]?.locale).toBe("en"); + expect(variants[0]?.messageId).toBe(messages[0]?.id); +}); diff --git a/inlang/source-code/sdk2/src/import-export/importFiles.ts b/inlang/source-code/sdk2/src/import-export/importFiles.ts index d46e720ecf..1931f51abd 100644 --- a/inlang/source-code/sdk2/src/import-export/importFiles.ts +++ b/inlang/source-code/sdk2/src/import-export/importFiles.ts @@ -5,8 +5,9 @@ import { } from "../plugin/errors.js"; import type { ProjectSettings } from "../json-schema/settings.js"; import type { InlangDatabaseSchema, NewVariant } from "../database/schema.js"; -import type { InlangPlugin } from "../plugin/schema.js"; +import type { InlangPlugin, VariantImport } from "../plugin/schema.js"; import type { ImportFile } from "../project/api.js"; +import { SQLite3Error } from "sqlite-wasm-kysely"; export async function importFiles(args: { files: ImportFile[]; @@ -53,23 +54,64 @@ export async function importFiles(args: { .executeTakeFirst(); message.id = exisingMessage?.id; } - await trx - .insertInto("message") - .values(message) - .onConflict((oc) => oc.column("id").doUpdateSet(message)) - .execute(); + try { + await trx + .insertInto("message") + .values(message) + .onConflict((oc) => oc.column("id").doUpdateSet(message)) + .execute(); + } catch (e) { + // 787 = SQLITE_CONSTRAINT_FOREIGNKEY + // handle foreign key violation + // e.g. a message references a bundle that doesn't exist + // by creating the bundle + if ((e as SQLite3Error)?.resultCode === 787) { + await trx + .insertInto("bundle") + .values({ id: message.bundleId }) + .execute(); + await trx.insertInto("message").values(message).execute(); + } else { + throw e; + } + } } // upsert every variant for (const variant of imported.variants) { // match the variant by message id and matches if // no id is provided by the importer if (variant.id === undefined) { - const existingMessage = await trx + let existingMessage = await trx .selectFrom("message") - .where("bundleId", "=", variant.bundleId) - .where("locale", "=", variant.locale) + .where("bundleId", "=", variant.messageBundleId) + .where("locale", "=", variant.messageLocale) .select("id") - .executeTakeFirstOrThrow(); + .executeTakeFirst(); + + // if the message does not exist, create it + if (existingMessage === undefined) { + const existingBundle = await trx + .selectFrom("bundle") + .where("id", "=", variant.messageBundleId) + .select("id") + .executeTakeFirst(); + // if the bundle does not exist, create it + if (existingBundle === undefined) { + await trx + .insertInto("bundle") + .values({ id: variant.messageBundleId }) + .execute(); + } + // insert the message + existingMessage = await trx + .insertInto("message") + .values({ + bundleId: variant.messageBundleId, + locale: variant.messageLocale, + }) + .returningAll() + .executeTakeFirstOrThrow(); + } const existingVariants = await trx .selectFrom("variant") @@ -81,14 +123,15 @@ export async function importFiles(args: { (v) => JSON.stringify(v.matches) === JSON.stringify(variant.matches) ); - variant.id = existingVariant?.id; - variant.messageId = existingMessage.id; + // need to reset typescript's type narrowing + (variant as VariantImport).id = existingVariant?.id; + (variant as VariantImport).messageId = existingMessage.id; } const toBeInsertedVariant: NewVariant = { ...variant, // @ts-expect-error - bundle id is provided by VariantImport but not needed when inserting - bundleId: undefined, - locale: undefined, + messageBundleId: undefined, + messageLocale: undefined, }; await trx .insertInto("variant") diff --git a/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts b/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts index 3a1523c08a..055ca986f9 100644 --- a/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts +++ b/inlang/source-code/sdk2/src/import-export/roundtrip.test.ts @@ -293,8 +293,8 @@ const mockPluginSimple: InlangPlugin = { selectors: [], }); variants.push({ - bundleId: key, - locale: file.locale, + messageBundleId: key, + messageLocale: file.locale, matches: [], pattern: [ { From 468313cfdc2acce6a0734b0aef3d47fcbddca2e6 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:26:39 -0400 Subject: [PATCH 06/11] fix types --- .../i18next/src/import-export/importFiles.ts | 8 ++++---- .../i18next/src/import-export/roundtrip.test.ts | 15 ++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts b/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts index 09bd82275b..9bb54cf96c 100644 --- a/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts +++ b/inlang/source-code/plugins/i18next/src/import-export/importFiles.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ -import type { Pattern, VariableReference, Variant } from "@inlang/sdk2" +import type { Bundle, Pattern, VariableReference, Variant } from "@inlang/sdk2" import { type plugin } from "../plugin.js" import { flatten } from "flat" import type { @@ -79,7 +79,7 @@ function parseMessage(args: { bundleId = `${args.namespace}:${bundleId}` } - const bundle: BundleImport = { + const bundle: Bundle = { id: bundleId, declarations: pattern.variableReferences.map((variableReference) => ({ type: "input-variable", @@ -94,8 +94,8 @@ function parseMessage(args: { } const variant: VariantImport = { - bundleId: bundleId, - locale: args.locale, + messageBundleId: bundleId, + messageLocale: args.locale, matches: [], pattern: pattern.result, } diff --git a/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts b/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts index 1f0f2cac04..90b8c608cb 100644 --- a/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts +++ b/inlang/source-code/plugins/i18next/src/import-export/roundtrip.test.ts @@ -310,10 +310,10 @@ test("keyWithObjectValue", async () => { expect(imported.bundles[1]?.id).toStrictEqual("keyWithObjectValue.valueB") expect( - imported.variants.find((v) => v.bundleId === "keyWithObjectValue.valueA")?.pattern + imported.variants.find((v) => v.messageBundleId === "keyWithObjectValue.valueA")?.pattern ).toStrictEqual([{ type: "text", value: "return this with valueB" }] satisfies Pattern) expect( - imported.variants.find((v) => v.bundleId === "keyWithObjectValue.valueB")?.pattern + imported.variants.find((v) => v.messageBundleId === "keyWithObjectValue.valueB")?.pattern ).toStrictEqual([{ type: "text", value: "more text" }] satisfies Pattern) }) @@ -329,10 +329,10 @@ test("keyWithArrayValue", async () => { expect(imported.bundles[1]?.id).toStrictEqual("keyWithArrayValue.1") expect( - imported.variants.find((v) => v.bundleId === "keyWithArrayValue.0")?.pattern + imported.variants.find((v) => v.messageBundleId === "keyWithArrayValue.0")?.pattern ).toStrictEqual([{ type: "text", value: "multiple" }] satisfies Pattern) expect( - imported.variants.find((v) => v.bundleId === "keyWithArrayValue.1")?.pattern + imported.variants.find((v) => v.messageBundleId === "keyWithArrayValue.1")?.pattern ).toStrictEqual([{ type: "text", value: "things" }] satisfies Pattern) }) @@ -467,7 +467,6 @@ test("it should put new entities into the file without a namespace", async () => const exported = await runExportFiles({ bundles: [...imported.bundles, newBundle], messages: [...imported.messages, newMessage], - //@ts-expect-error - variants are VariantImport which differs from the Variant type variants: [...imported.variants, newVariant], }) @@ -536,18 +535,20 @@ async function runExportFiles(imported: Awaited>) } for (const variant of imported.variants) { if (variant.id === undefined) { + // @ts-expect-error - variant is an VariantImport variant.id = `${Math.random() * 1000}` } if (variant.messageId === undefined) { + // @ts-expect-error - variant is an VariantImport variant.messageId = imported.messages.find( - (m: any) => m.bundleId === variant.bundleId && m.locale === variant.locale + (m: any) => m.bundleId === variant.messageBundleId && m.locale === variant.messageLocale )?.id } } const exported = await exportFiles({ settings: {} as any, - bundles: imported.bundles, + bundles: imported.bundles as Bundle[], messages: imported.messages as Message[], variants: imported.variants as Variant[], }) From 8681dfda8a3aafbab840f8c1afd1c9def84f7117 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:58:22 -0400 Subject: [PATCH 07/11] expose import types --- inlang/source-code/sdk2/src/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/inlang/source-code/sdk2/src/index.ts b/inlang/source-code/sdk2/src/index.ts index 489d670ab9..a2335f1f81 100644 --- a/inlang/source-code/sdk2/src/index.ts +++ b/inlang/source-code/sdk2/src/index.ts @@ -19,7 +19,12 @@ export * from "./plugin/errors.js"; export { humanId } from "./human-id/human-id.js"; export type { InlangDatabaseSchema } from "./database/schema.js"; export type { ImportFile, ExportFile } from "./project/api.js"; -export type { InlangPlugin } from "./plugin/schema.js"; +export type { + InlangPlugin, + BundleImport, + MessageImport, + VariantImport, +} from "./plugin/schema.js"; export type { IdeExtensionConfig } from "./plugin/meta/ideExtension.js"; export * from "./database/schema.js"; export * from "@lix-js/sdk"; From 94eb12639c8e90a14f528ecbd055e31cc46768a9 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:28:15 -0400 Subject: [PATCH 08/11] chore: add gitignore --- .../inlang-message-format/example/project.inlang/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 inlang/source-code/plugins/inlang-message-format/example/project.inlang/.gitignore diff --git a/inlang/source-code/plugins/inlang-message-format/example/project.inlang/.gitignore b/inlang/source-code/plugins/inlang-message-format/example/project.inlang/.gitignore new file mode 100644 index 0000000000..5e46596759 --- /dev/null +++ b/inlang/source-code/plugins/inlang-message-format/example/project.inlang/.gitignore @@ -0,0 +1 @@ +cache \ No newline at end of file From 46fa0dac0773607ff236c7ba5dbc5a2a49bff1b6 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:28:40 -0400 Subject: [PATCH 09/11] refactor to new import types --- .../src/import-export/exportFiles.test.ts | 306 -------------- .../src/import-export/exportFiles.ts | 30 +- .../src/import-export/importFiles.test.ts | 392 ------------------ .../src/import-export/importFiles.ts | 114 ++--- .../src/import-export/roundtrip.test.ts | 243 +++++++++++ .../inlang-message-format/src/plugin.test.ts | 126 ------ 6 files changed, 317 insertions(+), 894 deletions(-) delete mode 100644 inlang/source-code/plugins/inlang-message-format/src/import-export/exportFiles.test.ts delete mode 100644 inlang/source-code/plugins/inlang-message-format/src/import-export/importFiles.test.ts create mode 100644 inlang/source-code/plugins/inlang-message-format/src/import-export/roundtrip.test.ts delete mode 100644 inlang/source-code/plugins/inlang-message-format/src/plugin.test.ts diff --git a/inlang/source-code/plugins/inlang-message-format/src/import-export/exportFiles.test.ts b/inlang/source-code/plugins/inlang-message-format/src/import-export/exportFiles.test.ts deleted file mode 100644 index e6c533f1e9..0000000000 --- a/inlang/source-code/plugins/inlang-message-format/src/import-export/exportFiles.test.ts +++ /dev/null @@ -1,306 +0,0 @@ -import type { BundleNested, ProjectSettings } from "@inlang/sdk2" -import { expect, test } from "vitest" -import { PLUGIN_KEY } from "../plugin.js" -import type { PluginSettings } from "../settings.js" -import { exportFiles } from "./exportFiles.js" - -test("it handles single variants without expressions", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - [PLUGIN_KEY]: { - pathPattern: "/i18n/{locale}.json", - } satisfies PluginSettings, - } - - const mockEnFile = { - some_happy_cat: "Read more about Lix", - } - - const mockBundle = { - id: "some_happy_cat", - declarations: [], - messages: [ - { - id: "some_happy_cat_en", - bundleId: "some_happy_cat", - locale: "en", - selectors: [], - variants: [ - { - id: "some_happy_cat_en", - matches: [], - messageId: "some_happy_cat_en", - pattern: [{ type: "text", value: "Read more about Lix" }], - }, - ], - }, - ], - } satisfies BundleNested - - const result = await exportFiles({ - settings: mockSettings, - bundles: [mockBundle], - }) - - expect(result).lengthOf(1) - expect(result[0]?.name).toBe("en.json") - expect(result[0]?.locale).toBe("en") - - const parsed = JSON.parse(new TextDecoder().decode(result[0]?.content)) - - expect(parsed).toStrictEqual(mockEnFile) -}) - -test("it handles multi variants", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - [PLUGIN_KEY]: { - pathPattern: "/i18n/{locale}.json", - } satisfies PluginSettings, - } - - const mockEnFile = { - some_happy_cat: { - match: { - "platform=android, userGender=male": - "{username} has to download the app on his phone from the Google Play Store.", - "platform=ios, userGender=female": - "{username} has to download the app on her iPhone from the App Store.", - "platform=*, userGender=*": "The person has to download the app.", - }, - }, - } - - const mockBundle: BundleNested = { - id: "some_happy_cat", - declarations: [], - messages: [ - { - id: "some_happy_cat_en", - bundleId: "some_happy_cat", - locale: "en", - selectors: [], - variants: [ - { - id: "some_happy_cat_en;platform=android,userGender=male", - matches: [ - { - type: "literal-match", - key: "platform", - value: "android", - }, - { - type: "literal-match", - key: "userGender", - value: "male", - }, - ], - messageId: "some_happy_cat_en", - pattern: [ - { - type: "expression", - arg: { type: "variable-reference", name: "username" }, - }, - { - type: "text", - value: " has to download the app on his phone from the Google Play Store.", - }, - ], - }, - { - id: "some_happy_cat_en;platform=ios,userGender=female", - matches: [ - { - type: "literal-match", - key: "platform", - value: "ios", - }, - { - type: "literal-match", - key: "userGender", - value: "female", - }, - ], - messageId: "some_happy_cat_en", - pattern: [ - { - type: "expression", - arg: { type: "variable-reference", name: "username" }, - }, - { - type: "text", - value: " has to download the app on her iPhone from the App Store.", - }, - ], - }, - { - id: "some_happy_cat_en;platform=*,userGender=*", - matches: [ - { - type: "catchall-match", - key: "platform", - }, - { - type: "catchall-match", - key: "userGender", - }, - ], - messageId: "some_happy_cat_en", - pattern: [ - { - type: "text", - value: "The person has to download the app.", - }, - ], - }, - ], - }, - ], - } - - const result = await exportFiles({ - settings: mockSettings, - bundles: [mockBundle], - }) - - expect(result).lengthOf(1) - expect(result[0]?.name).toBe("en.json") - expect(result[0]?.locale).toBe("en") - - const parsed = JSON.parse(new TextDecoder().decode(result[0]?.content)) - - expect(parsed).toStrictEqual(mockEnFile) -}) - -test("it handles variable expressions in patterns", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - [PLUGIN_KEY]: { - pathPattern: "/i18n/{locale}.json", - } satisfies PluginSettings, - } - - const mockEnFile = { - some_happy_cat: "Used by {count} devs, {numDesigners} designers and translators", - } - - const mockBundle = { - id: "some_happy_cat", - declarations: [], - messages: [ - { - id: "some_happy_cat_en", - bundleId: "some_happy_cat", - locale: "en", - selectors: [], - variants: [ - { - id: "some_happy_cat_en", - matches: [], - messageId: "some_happy_cat_en", - pattern: [ - { type: "text", value: "Used by " }, - { type: "expression", arg: { type: "variable-reference", name: "count" } }, - { type: "text", value: " devs, " }, - { type: "expression", arg: { type: "variable-reference", name: "numDesigners" } }, - { type: "text", value: " designers and translators" }, - ], - }, - ], - }, - ], - } satisfies BundleNested - - const result = await exportFiles({ - settings: mockSettings, - bundles: [mockBundle], - }) - - expect(result).lengthOf(1) - expect(result[0]?.name).toBe("en.json") - expect(result[0]?.locale).toBe("en") - - const parsed = JSON.parse(new TextDecoder().decode(result[0]?.content)) - - expect(parsed).toStrictEqual(mockEnFile) -}) - -test("it assigns the correct locales to files", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - [PLUGIN_KEY]: { - pathPattern: "/i18n/{locale}.json", - } satisfies PluginSettings, - } - - const mockBundle: BundleNested = { - id: "some_happy_cat", - declarations: [], - messages: [ - { - id: "some_happy_cat_en", - bundleId: "some_happy_cat", - locale: "en", - selectors: [], - variants: [ - { - id: "some_happy_cat_en", - matches: [], - messageId: "some_happy_cat_en", - pattern: [{ type: "text", value: "Read more about Lix" }], - }, - ], - }, - { - id: "some_happy_cat_de", - bundleId: "some_happy_cat", - locale: "de", - selectors: [], - variants: [ - { - id: "some_happy_cat_de", - matches: [], - messageId: "some_happy_cat_de", - pattern: [{ type: "text", value: "Lese mehr über Lix" }], - }, - ], - }, - ], - } - - const result = await exportFiles({ - settings: mockSettings, - bundles: [mockBundle], - }) - - expect(result).lengthOf(2) - expect(result).toStrictEqual([ - expect.objectContaining({ - locale: "en", - name: "en.json", - }), - expect.objectContaining({ - locale: "de", - name: "de.json", - }), - ]) -}) - -test("it should throw if pathPattern is not defined because no export is possible", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - [PLUGIN_KEY]: {}, - } - - await expect( - exportFiles({ - settings: mockSettings, - bundles: [], - }) - ).rejects.toThrow("pathPattern is not defined") -}) diff --git a/inlang/source-code/plugins/inlang-message-format/src/import-export/exportFiles.ts b/inlang/source-code/plugins/inlang-message-format/src/import-export/exportFiles.ts index f75602d105..cfb580af0b 100644 --- a/inlang/source-code/plugins/inlang-message-format/src/import-export/exportFiles.ts +++ b/inlang/source-code/plugins/inlang-message-format/src/import-export/exportFiles.ts @@ -1,21 +1,19 @@ -import type { ExportFile, Match, MessageNested, Variant } from "@inlang/sdk2" -import { PLUGIN_KEY, type plugin } from "../plugin.js" +import type { ExportFile, Match, Message, Variant } from "@inlang/sdk2" +import { type plugin } from "../plugin.js" import type { FileSchema } from "../fileSchema.js" export const exportFiles: NonNullable<(typeof plugin)["exportFiles"]> = async ({ - bundles, - settings, + messages, + variants, }) => { - const pathPattern = settings[PLUGIN_KEY]?.pathPattern - if (pathPattern === undefined) { - throw new Error("pathPattern is not defined") - } - const files: Record = {} - const messages = bundles.flatMap((bundle) => bundle.messages) for (const message of messages) { - files[message.locale] = { ...files[message.locale], ...serializeMessage(message) } + const variantsOfMessage = variants.filter((v) => v.messageId === message.id) + files[message.locale] = { + ...files[message.locale], + ...serializeMessage(message, variantsOfMessage), + } } const result: ExportFile[] = [] @@ -32,11 +30,12 @@ export const exportFiles: NonNullable<(typeof plugin)["exportFiles"]> = async ({ return result } -function serializeMessage(message: MessageNested): { - [key: string]: string | Record -} { +function serializeMessage( + message: Message, + variants: Variant[] +): Record> { const key = message.bundleId - const value = serializeVariants(message.variants) + const value = serializeVariants(variants) return { [key]: value } } @@ -45,6 +44,7 @@ function serializeVariants(variants: Variant[]): string | Record // todo add logic for handling if a variant has a match even if it's // the only variant if (variants.length === 1) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion return serializePattern(variants[0]!.pattern) } const entries = [] diff --git a/inlang/source-code/plugins/inlang-message-format/src/import-export/importFiles.test.ts b/inlang/source-code/plugins/inlang-message-format/src/import-export/importFiles.test.ts deleted file mode 100644 index c6cd9c41be..0000000000 --- a/inlang/source-code/plugins/inlang-message-format/src/import-export/importFiles.test.ts +++ /dev/null @@ -1,392 +0,0 @@ -import type { BundleNested, ProjectSettings } from "@inlang/sdk2" -import { expect, test } from "vitest" -import { importFiles } from "./importFiles.js" - -test("it handles single variants without expressions", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - } - - const mockEnFile = new TextEncoder().encode( - JSON.stringify({ - some_happy_cat: "Read more about Lix", - }) - ) - - const result = await importFiles({ - settings: mockSettings, - files: [ - { - locale: "en", - content: mockEnFile, - }, - ], - }) - - expect(result).toStrictEqual({ - bundles: [ - { - id: "some_happy_cat", - declarations: [], - messages: [ - { - id: "some_happy_cat_en", - bundleId: "some_happy_cat", - locale: "en", - selectors: [], - variants: [ - { - id: "some_happy_cat_en", - matches: [], - messageId: "some_happy_cat_en", - pattern: [{ type: "text", value: "Read more about Lix" }], - }, - ], - }, - ], - }, - ] satisfies BundleNested[], - }) -}) - -test("it handles variable expressions in patterns", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - } - - const mockEnFile = new TextEncoder().encode( - JSON.stringify({ - some_happy_cat: "Used by {count} devs, {numDesigners} designers and translators", - }) - ) - - const result = await importFiles({ - settings: mockSettings, - files: [ - { - locale: "en", - content: mockEnFile, - }, - ], - }) - - expect(result).toStrictEqual({ - bundles: [ - { - id: "some_happy_cat", - declarations: [], - messages: [ - { - id: "some_happy_cat_en", - bundleId: "some_happy_cat", - locale: "en", - selectors: [], - variants: [ - { - id: "some_happy_cat_en", - matches: [], - messageId: "some_happy_cat_en", - pattern: [ - { type: "text", value: "Used by " }, - { - type: "expression", - arg: { type: "variable-reference", name: "count" }, - }, - { - type: "text", - value: " devs, ", - }, - { type: "expression", arg: { type: "variable-reference", name: "numDesigners" } }, - { - type: "text", - value: " designers and translators", - }, - ], - }, - ], - }, - ], - }, - ] satisfies BundleNested[], - }) -}) - -test("it ingores the $schema property that is used for typesafety", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - } - - const mockEnFile = new TextEncoder().encode( - JSON.stringify({ - $schema: "https://mock.com/file-schema", - }) - ) - - const result = await importFiles({ - settings: mockSettings, - files: [ - { - locale: "en", - content: mockEnFile, - }, - ], - }) - - expect(result).toStrictEqual({ - bundles: [], - }) -}) - -test("it assigns the correct locales to messages", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - } - - const mockEnFile = new TextEncoder().encode( - JSON.stringify({ - some_happy_cat: "Read more about Lix", - }) - ) - - const mockDeFile = new TextEncoder().encode( - JSON.stringify({ - some_happy_cat: "Lese mehr über Lix", - }) - ) - - const result = await importFiles({ - settings: mockSettings, - files: [ - { - locale: "en", - content: mockEnFile, - }, - { - locale: "de", - content: mockDeFile, - }, - ], - }) - - expect(result).toStrictEqual({ - bundles: [ - { - id: "some_happy_cat", - declarations: [], - messages: [ - { - id: "some_happy_cat_en", - bundleId: "some_happy_cat", - locale: "en", - selectors: [], - variants: [ - { - id: "some_happy_cat_en", - matches: [], - messageId: "some_happy_cat_en", - pattern: [{ type: "text", value: "Read more about Lix" }], - }, - ], - }, - { - id: "some_happy_cat_de", - bundleId: "some_happy_cat", - locale: "de", - selectors: [], - variants: [ - { - id: "some_happy_cat_de", - matches: [], - messageId: "some_happy_cat_de", - pattern: [{ type: "text", value: "Lese mehr über Lix" }], - }, - ], - }, - ], - }, - ] satisfies BundleNested[], - }) -}) - -test("it handles multi variant messages", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - } - - const mockEnFile = new TextEncoder().encode( - JSON.stringify({ - some_happy_cat: { - match: { - "platform=android, userGender=male": - "{username} has to download the app on his phone from the Google Play Store.", - "platform=ios, userGender=female": - "{username} has to download the app on her iPhone from the App Store.", - "platform=*, userGender=*": "The person has to download the app.", - }, - }, - }) - ) - - const result = await importFiles({ - settings: mockSettings, - files: [ - { - locale: "en", - content: mockEnFile, - }, - ], - }) - - expect(result).toStrictEqual({ - bundles: [ - { - id: "some_happy_cat", - declarations: [], - messages: [ - { - id: "some_happy_cat_en", - bundleId: "some_happy_cat", - locale: "en", - selectors: [], - variants: [ - { - id: "some_happy_cat_en;platform=android,userGender=male", - matches: [ - { - type: "literal-match", - key: "platform", - value: "android", - }, - { - type: "literal-match", - key: "userGender", - value: "male", - }, - ], - messageId: "some_happy_cat_en", - pattern: [ - { - type: "expression", - arg: { type: "variable-reference", name: "username" }, - }, - { - type: "text", - value: " has to download the app on his phone from the Google Play Store.", - }, - ], - }, - { - id: "some_happy_cat_en;platform=ios,userGender=female", - matches: [ - { - type: "literal-match", - key: "platform", - value: "ios", - }, - { - type: "literal-match", - key: "userGender", - value: "female", - }, - ], - messageId: "some_happy_cat_en", - pattern: [ - { - type: "expression", - arg: { type: "variable-reference", name: "username" }, - }, - { - type: "text", - value: " has to download the app on her iPhone from the App Store.", - }, - ], - }, - { - id: "some_happy_cat_en;platform=*,userGender=*", - matches: [ - { - type: "catchall-match", - key: "platform", - }, - { - type: "catchall-match", - key: "userGender", - }, - ], - messageId: "some_happy_cat_en", - pattern: [ - { - type: "text", - value: "The person has to download the app.", - }, - ], - }, - ], - }, - ], - }, - ] satisfies BundleNested[], - }) -}) - -// test.todo("it parses multi variants", async () => { -// const mockFile: FileSchema = { -// some_happy_cat: { -// selectors: { -// count: "plural(count)", -// }, -// match: { -// "count=one": "You have one photo.", -// "count=other": "You have {count} photos.", -// }, -// }, -// multi_selector: { -// "plural(photoCount)=other, plural(likeCount)=one": "You have one photo and one like.", -// "plural(photoCount)=one, plural(likeCount)=one": "You have one photo and one like.", -// }, -// } - -// const result = parseFile({ -// locale: "en", -// content: new TextEncoder().encode(JSON.stringify(mockFile)), -// }) - -// expect(result).toStrictEqual([ -// { -// id: "some_happy_cat_en", -// bundleId: "some_happy_cat", -// locale: "en", -// selectors: [], -// declarations: [], -// variants: [ -// { -// id: "some_happy_cat_en", -// match: {}, -// messageId: "some_happy_cat_en", -// pattern: [ -// { type: "text", value: "Used by " }, -// { -// type: "expression", -// arg: { type: "variable", name: "count" }, -// }, -// { -// type: "text", -// value: " devs, ", -// }, -// { type: "expression", arg: { type: "variable", name: "numDesigners" } }, -// { -// type: "text", -// value: " designers and translators", -// }, -// ], -// }, -// ], -// }, -// ] satisfies MessageNested[]) -// }) -// diff --git a/inlang/source-code/plugins/inlang-message-format/src/import-export/importFiles.ts b/inlang/source-code/plugins/inlang-message-format/src/import-export/importFiles.ts index 8fb238f347..1a6d7b5f3e 100644 --- a/inlang/source-code/plugins/inlang-message-format/src/import-export/importFiles.ts +++ b/inlang/source-code/plugins/inlang-message-format/src/import-export/importFiles.ts @@ -1,85 +1,86 @@ -import { type Match, type MessageNested, type NewBundleNested, type Variant } from "@inlang/sdk2" +import type { Match, Variant, MessageImport, VariantImport, Bundle } from "@inlang/sdk2" import { type plugin } from "../plugin.js" export const importFiles: NonNullable<(typeof plugin)["importFiles"]> = async ({ files }) => { - const messages = files.flatMap(parseFile) - const bundlesIndex: Record = {} + const bundles: Bundle[] = [] + const messages: MessageImport[] = [] + const variants: VariantImport[] = [] - for (const message of messages) { - if (bundlesIndex[message.bundleId] === undefined) { - bundlesIndex[message.bundleId] = { - id: message.bundleId, - declarations: [], - messages: [message], - } - } else { - bundlesIndex[message.bundleId].messages.push(message) - } - } - - const bundles = Object.values(bundlesIndex) as NewBundleNested[] - - return { bundles } -} - -function parseFile(args: { locale: string; content: ArrayBuffer }): MessageNested[] { - const json = JSON.parse(new TextDecoder().decode(args.content)) + for (const file of files) { + const json = JSON.parse(new TextDecoder().decode(file.content)) - const messages: MessageNested[] = [] + for (const key in json) { + if (key === "$schema") { + continue + } + const result = parseMessage(key, file.locale, json[key]) + messages.push(result.message) + variants.push(...result.variants) - for (const key in json) { - if (key === "$schema") { - continue + const existingBundle = bundles.find((b) => b.id === result.bundle.id) + if (existingBundle === undefined) { + bundles.push(result.bundle) + } else { + // merge declarations without duplicates + existingBundle.declarations = removeDuplicates([ + existingBundle.declarations, + ...result.bundle.declarations, + ]) + } } - messages.push( - parseMessage({ - key, - value: json[key], - locale: args.locale, - }) - ) } - return messages + + return { bundles, messages, variants } } -function parseMessage(args: { - key: string +function parseMessage( + key: string, + locale: string, value: string | Record - locale: string -}): MessageNested { - // happy_sky_eye + _ + en - const messageId = args.key + "_" + args.locale - const variants = parseVariants(messageId, args.value) +): { + bundle: Bundle + message: MessageImport + variants: VariantImport[] +} { + const variants = parseVariants(key, locale, value) return { - id: messageId, - bundleId: args.key, - selectors: [], - locale: args.locale, + bundle: { + id: key, + declarations: [], + }, + message: { + bundleId: key, + selectors: [], + locale: locale, + }, variants, } } -function parseVariants(messageId: string, value: string | Record): Variant[] { +function parseVariants( + bundleId: string, + locale: string, + value: string | Record +): VariantImport[] { // single variant if (typeof value === "string") { return [ { - id: messageId, - messageId, + messageBundleId: bundleId, + messageLocale: locale, matches: [], pattern: parsePattern(value), }, ] } // multi variant - const variants: Variant[] = [] - for (const [matcher, pattern] of Object.entries(value["match"] as string)) { + const variants: VariantImport[] = [] + for (const [match, pattern] of Object.entries(value["match"] as string)) { variants.push({ - // "some_happy_cat_en;platform=ios,userGender=female" - id: messageId + ";" + matcher.replaceAll(" ", ""), - messageId, - matches: parseMatcher(matcher), + messageBundleId: bundleId, + messageLocale: locale, + matches: parseMatch(match), pattern: parsePattern(pattern), }) } @@ -110,7 +111,7 @@ function parsePattern(value: string): Variant["pattern"] { // input: `platform=android,userGender=male` // output: { platform: "android", userGender: "male" } -function parseMatcher(value: string): Match[] { +function parseMatch(value: string): Match[] { const stripped = value.replace(" ", "") const matches: Match[] = [] const parts = stripped.split(",") @@ -134,3 +135,6 @@ function parseMatcher(value: string): Match[] { } return matches } + +const removeDuplicates = (arr: T) => + [...new Set(arr.map((item) => JSON.stringify(item)))].map((item) => JSON.parse(item)) diff --git a/inlang/source-code/plugins/inlang-message-format/src/import-export/roundtrip.test.ts b/inlang/source-code/plugins/inlang-message-format/src/import-export/roundtrip.test.ts new file mode 100644 index 0000000000..184d54e710 --- /dev/null +++ b/inlang/source-code/plugins/inlang-message-format/src/import-export/roundtrip.test.ts @@ -0,0 +1,243 @@ +import { expect, test } from "vitest" +import { importFiles } from "./importFiles.js" +import { type Bundle, type Message, type Pattern, type Variant } from "@inlang/sdk2" +import { exportFiles } from "./exportFiles.js" + +test("it handles single variants without expressions", async () => { + const imported = await runImportFiles({ + some_happy_cat: "Read more about Lix", + }) + expect(await runExportFilesParsed(imported)).toStrictEqual({ + some_happy_cat: "Read more about Lix", + }) + + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(1) + + expect(imported.bundles[0]?.id).toStrictEqual("some_happy_cat") + expect(imported.bundles[0]?.declarations).toStrictEqual([]) + + expect(imported.messages[0]?.selectors).toStrictEqual([]) + + expect(imported.variants[0]?.matches).toStrictEqual([]) + expect(imported.variants[0]?.pattern).toStrictEqual([ + { type: "text", value: "Read more about Lix" }, + ]) +}) + +test("it handles variable expressions in patterns", async () => { + const imported = await runImportFiles({ + some_happy_cat: "Used by {count} devs, {numDesigners} designers and translators", + }) + expect(await runExportFilesParsed(imported)).toStrictEqual({ + some_happy_cat: "Used by {count} devs, {numDesigners} designers and translators", + }) + + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(1) + + expect(imported.bundles[0]?.id).toStrictEqual("some_happy_cat") + expect(imported.bundles[0]?.declarations).toStrictEqual([]) + + expect(imported.messages[0]?.selectors).toStrictEqual([]) + + expect(imported.variants[0]?.matches).toStrictEqual([]) + expect(imported.variants[0]?.pattern).toStrictEqual([ + { type: "text", value: "Used by " }, + { + type: "expression", + arg: { type: "variable-reference", name: "count" }, + }, + { + type: "text", + value: " devs, ", + }, + { type: "expression", arg: { type: "variable-reference", name: "numDesigners" } }, + { + type: "text", + value: " designers and translators", + }, + ] satisfies Pattern) +}) + +test("it removes the $schema property (to reduce inter-dependencies and the requirement to have a server)", async () => { + const imported = await runImportFiles({ + $schema: "https://mock.com/file-schema", + key: "value", + }) + expect(await runExportFilesParsed(imported)).toStrictEqual({ + key: "value", + }) +}) + +test("it handles multi variant messages", async () => { + const imported = await runImportFiles({ + some_happy_cat: { + match: { + "platform=android, userGender=male": + "{username} has to download the app on his phone from the Google Play Store.", + "platform=ios, userGender=female": + "{username} has to download the app on her iPhone from the App Store.", + "platform=*, userGender=*": "The person has to download the app.", + }, + }, + }) + expect(await runExportFilesParsed(imported)).toStrictEqual({ + some_happy_cat: { + match: { + "platform=android, userGender=male": + "{username} has to download the app on his phone from the Google Play Store.", + "platform=ios, userGender=female": + "{username} has to download the app on her iPhone from the App Store.", + "platform=*, userGender=*": "The person has to download the app.", + }, + }, + }) + + expect(imported.bundles).lengthOf(1) + expect(imported.messages).lengthOf(1) + expect(imported.variants).lengthOf(3) + + expect(imported.bundles[0]?.id).toStrictEqual("some_happy_cat") + expect(imported.bundles[0]?.declarations).toStrictEqual([]) + + expect(imported.messages[0]?.selectors).toStrictEqual([]) + expect(imported.messages[0]?.bundleId).toStrictEqual("some_happy_cat") + + expect(imported.variants[0]).toStrictEqual( + expect.objectContaining({ + matches: [ + { type: "literal-match", key: "platform", value: "android" }, + { type: "literal-match", key: "userGender", value: "male" }, + ], + pattern: [ + { type: "expression", arg: { type: "variable-reference", name: "username" } }, + { + type: "text", + value: " has to download the app on his phone from the Google Play Store.", + }, + ], + } satisfies Partial) + ) + expect(imported.variants[1]).toStrictEqual( + expect.objectContaining({ + matches: [ + { type: "literal-match", key: "platform", value: "ios" }, + { type: "literal-match", key: "userGender", value: "female" }, + ], + pattern: [ + { type: "expression", arg: { type: "variable-reference", name: "username" } }, + { + type: "text", + value: " has to download the app on her iPhone from the App Store.", + }, + ], + } satisfies Partial) + ) + expect(imported.variants[2]).toStrictEqual( + expect.objectContaining({ + matches: [ + { type: "catchall-match", key: "platform" }, + { type: "catchall-match", key: "userGender" }, + ], + pattern: [{ type: "text", value: "The person has to download the app." }], + } satisfies Partial) + ) +}) + +test("roundtrip with new variants that have been created by apps", async () => { + const imported1 = await runImportFiles({ + some_happy_cat: "Read more about Lix", + }) + + // simulating adding a new bundle, message, and variant + imported1.bundles.push({ + id: "green_box_atari", + declarations: [], + }) + + imported1.messages.push({ + id: "0j299j-3si02j0j4=s02-3js2", + bundleId: "green_box_atari", + selectors: [], + locale: "en", + }) + + imported1.variants.push({ + id: "929s", + matches: [], + messageId: "0j299j-3si02j0j4=s02-3js2", + pattern: [{ type: "text", value: "New variant" }], + }) + + // export after adding the bundle, messages, variants + const exported1 = await runExportFiles(imported1) + + const imported2 = await runImportFiles( + JSON.parse(new TextDecoder().decode(exported1[0]?.content)) + ) + + const exported2 = await runExportFiles(imported2) + + expect(imported2.bundles).toStrictEqual([ + expect.objectContaining({ + id: "some_happy_cat", + }), + expect.objectContaining({ + id: "green_box_atari", + }), + ]) + + expect(exported2).toStrictEqual(exported1) +}) + +// convenience wrapper for less testing code +function runImportFiles(json: Record) { + return importFiles({ + settings: {} as any, + files: [ + { + locale: "en", + content: new TextEncoder().encode(JSON.stringify(json)), + }, + ], + }) +} + +// convenience wrapper for less testing code +async function runExportFiles(imported: Awaited>) { + // add ids which are undefined from the import + for (const message of imported.messages) { + if (message.id === undefined) { + message.id = `${Math.random() * 1000}` + } + } + for (const variant of imported.variants) { + if (variant.id === undefined) { + // @ts-expect-error - variant is an VariantImport + variant.id = `${Math.random() * 1000}` + } + if (variant.messageId === undefined) { + // @ts-expect-error - variant is an VariantImport + variant.messageId = imported.messages.find( + (m: any) => m.bundleId === variant.messageBundleId && m.locale === variant.messageLocale + )?.id + } + } + + const exported = await exportFiles({ + settings: {} as any, + bundles: imported.bundles as Bundle[], + messages: imported.messages as Message[], + variants: imported.variants as Variant[], + }) + return exported +} + +// convenience wrapper for less testing code +async function runExportFilesParsed(imported: any) { + const exported = await runExportFiles(imported) + return JSON.parse(new TextDecoder().decode(exported[0]?.content)) +} diff --git a/inlang/source-code/plugins/inlang-message-format/src/plugin.test.ts b/inlang/source-code/plugins/inlang-message-format/src/plugin.test.ts deleted file mode 100644 index 94caa3dec1..0000000000 --- a/inlang/source-code/plugins/inlang-message-format/src/plugin.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import type { BundleNested, ProjectSettings } from "@inlang/sdk2" -import { test, expect } from "vitest" -import { PLUGIN_KEY } from "./plugin.js" -import type { PluginSettings } from "./settings.js" -import { importFiles } from "./import-export/importFiles.js" -import { exportFiles } from "./import-export/exportFiles.js" - -test("roundtrip of import and export", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - [PLUGIN_KEY]: { - pathPattern: "/i18n/{locale}.json", - } satisfies PluginSettings, - } - - const mockEnFileParsed = { - some_happy_cat: "Read more about Lix", - blue_horse_shoe: "Hello {username}, welcome to the {placename}!", - jojo_mountain_day: { - match: { - "platform=android, userGender=male": - "{username} has to download the app on his phone from the Google Play Store.", - "platform=ios, userGender=female": - "{username} has to download the app on her iPhone from the App Store.", - "platform=*, userGender=*": "The person has to download the app.", - }, - }, - } - - const imported = await importFiles({ - settings: mockSettings, - files: [ - { - content: new TextEncoder().encode(JSON.stringify(mockEnFileParsed)), - locale: "en", - }, - ], - }) - - const exported = await exportFiles({ - settings: mockSettings, - bundles: imported.bundles as BundleNested[], - }) - - expect(exported).lengthOf(1) - expect(exported[0]?.name).toBe("en.json") - expect(exported[0]?.locale).toBe("en") - - const parsed = JSON.parse(new TextDecoder().decode(exported[0]?.content)) - - expect(parsed).toStrictEqual(mockEnFileParsed) -}) - -test("roundtrip with new variants that have been created by apps", async () => { - const mockSettings: ProjectSettings = { - baseLocale: "en", - locales: ["en", "de"], - [PLUGIN_KEY]: { - pathPattern: "/i18n/{locale}.json", - } satisfies PluginSettings, - } - - const mockEnFileParsed = { - some_happy_cat: "Read more about Lix", - } - - const imported1 = await importFiles({ - settings: mockSettings, - files: [ - { - content: new TextEncoder().encode(JSON.stringify(mockEnFileParsed)), - locale: "en", - }, - ], - }) - - // simulating adding a new bundle, message, and variant - imported1.bundles.push({ - id: "green_box_atari", - declarations: [], - messages: [ - { - id: "0j299j-3si02j0j4=s02-3js2", - bundleId: "green_box_atari", - selectors: [], - locale: "en", - variants: [ - { - id: "929s", - matches: [], - messageId: "0j299j-3si02j0j4=s02-3js2", - pattern: [{ type: "text", value: "New variant" }], - }, - ], - }, - ], - }) - - // export after adding the bundle, messages, variants - const exported1 = await exportFiles({ - settings: mockSettings, - bundles: imported1.bundles as BundleNested[], - }) - - const imported2 = await importFiles({ - settings: mockSettings, - files: exported1, - }) - - const exported2 = await exportFiles({ - settings: mockSettings, - bundles: imported2.bundles as BundleNested[], - }) - - expect(imported2.bundles).toStrictEqual([ - expect.objectContaining({ - id: "some_happy_cat", - }), - expect.objectContaining({ - id: "green_box_atari", - }), - ]) - - expect(exported2).toStrictEqual(exported1) -}) From 63e49f22f83f519194aee1c7bb6bdc9e8466e166 Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:30:30 -0400 Subject: [PATCH 10/11] Revert "expose sqlite3 error" This reverts commit dde3b7474b650a98d2af7cda4872c5d773109851. --- lix/packages/sqlite-wasm-kysely/src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lix/packages/sqlite-wasm-kysely/src/index.ts b/lix/packages/sqlite-wasm-kysely/src/index.ts index 629d38d9bb..3719ce6f33 100644 --- a/lix/packages/sqlite-wasm-kysely/src/index.ts +++ b/lix/packages/sqlite-wasm-kysely/src/index.ts @@ -1,3 +1,2 @@ -export { SQLite3Error } from "@eliaspourquoi/sqlite-node-wasm"; export * from "./util/index.js"; export { createDialect } from "./dialect.js"; From 99bbeaf5e6bea0f0c73c9a1435ff7e06836566ae Mon Sep 17 00:00:00 2001 From: Samuel Stroschein <35429197+samuelstroschein@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:34:12 -0400 Subject: [PATCH 11/11] fix: remove flaky import --- inlang/source-code/sdk2/src/import-export/importFiles.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/inlang/source-code/sdk2/src/import-export/importFiles.ts b/inlang/source-code/sdk2/src/import-export/importFiles.ts index 1931f51abd..cebbab84a7 100644 --- a/inlang/source-code/sdk2/src/import-export/importFiles.ts +++ b/inlang/source-code/sdk2/src/import-export/importFiles.ts @@ -7,7 +7,6 @@ import type { ProjectSettings } from "../json-schema/settings.js"; import type { InlangDatabaseSchema, NewVariant } from "../database/schema.js"; import type { InlangPlugin, VariantImport } from "../plugin/schema.js"; import type { ImportFile } from "../project/api.js"; -import { SQLite3Error } from "sqlite-wasm-kysely"; export async function importFiles(args: { files: ImportFile[]; @@ -65,7 +64,7 @@ export async function importFiles(args: { // handle foreign key violation // e.g. a message references a bundle that doesn't exist // by creating the bundle - if ((e as SQLite3Error)?.resultCode === 787) { + if ((e as any)?.resultCode === 787) { await trx .insertInto("bundle") .values({ id: message.bundleId })