Add unicode support to alphaTex (CoderLine#580)

fanjink · Apr 3, 2021 · 1f3f956 · 1f3f956
1 parent bae294d
commit 1f3f956
Show file tree

Hide file tree

Showing 13 changed files with 54 additions and 43 deletions.
diff --git a/src.compiler/csharp/CSharpAstPrinter.ts b/src.compiler/csharp/CSharpAstPrinter.ts
@@ -612,7 +612,7 @@ export default class CSharpAstPrinter extends AstPrinterBase {
         let exprs: cs.Expression[] = [];
         expr.chunks.forEach(c => {
             if (cs.isStringLiteral(c)) {
-                const escapedText = c.text.split('"').join('""').split('\n').join('\\n').split('\r').join('\\r');
+                const escapedText = c.text.split('"').join('""');
                 this.write(escapedText);
             } else {
                 this.write(`{${exprs.length}}`);

diff --git a/src.compiler/kotlin/KotlinAstPrinter.ts b/src.compiler/kotlin/KotlinAstPrinter.ts
@@ -858,18 +858,18 @@ export default class KotlinAstPrinter extends AstPrinterBase {
     }
 
     protected writeStringTemplateExpression(expr: cs.StringTemplateExpression) {
-        this.write('"');
+        this.write('"""');
         expr.chunks.forEach(c => {
             if (cs.isStringLiteral(c)) {
-                const escapedText = c.text.split('"').join('\\"').split('\n').join('\\n').split('\r').join('\\r');
+                const escapedText = c.text;
                 this.write(escapedText);
             } else {
                 this.write('${');
                 this.writeExpression(c);
                 this.write('}');
             }
         });
-        this.write('"');
+        this.write('"""');
     }
 
     protected writeArrayCreationExpression(expr: cs.ArrayCreationExpression) {

diff --git a/src.csharp/AlphaTab/AlphaTab.csproj b/src.csharp/AlphaTab/AlphaTab.csproj
@@ -11,8 +11,6 @@
         <TargetFramework>netstandard20</TargetFramework>
         <Nullable>enable</Nullable>
         <LangVersion>8</LangVersion>
-        <DebugSymbols>true</DebugSymbols>
-        <DebugType>full</DebugType>
     </PropertyGroup>
 
     <ItemGroup>

diff --git a/src.csharp/Directory.Build.props b/src.csharp/Directory.Build.props
@@ -2,15 +2,15 @@
   <PropertyGroup>
     <DebugType>portable</DebugType>
     <DebugSymbols>true</DebugSymbols>
-    <Version>1.1.0</Version>
-    <AssemblyVersion>1.1.0.0</AssemblyVersion>
+    <Version>1.3.0</Version>
+    <AssemblyVersion>1.3.0.0</AssemblyVersion>
     <FileVersion>$(AssemblyVersion)</FileVersion>
     <Authors>Danielku15</Authors>
     <Company>CoderLine</Company>
     <Product>AlphaTab</Product>
     <NeutralLanguage>en</NeutralLanguage>
     <Description>alphaTab is a cross platform music notation and guitar tablature rendering library.</Description>
-    <Copyright>Copyright © 2020, Daniel Kuschny and Contributors</Copyright>
+    <Copyright>Copyright © 2021, Daniel Kuschny and Contributors</Copyright>
     <PackageLicenseExpression>MPL-2.0</PackageLicenseExpression>
     <PackageProjectUrl>https://www.alphatab.net</PackageProjectUrl>
     <RepositoryUrl>https://github.com/CoderLine/alphaTab</RepositoryUrl>

diff --git a/src/AlphaTabApiBase.ts b/src/AlphaTabApiBase.ts
@@ -13,7 +13,6 @@ import { EventEmitter, IEventEmitter, IEventEmitterOfT, EventEmitterOfT } from '
 
 import { AlphaTexImporter } from '@src/importer/AlphaTexImporter';
 
-import { ByteBuffer } from '@src/io/ByteBuffer';
 import { Beat } from '@src/model/Beat';
 import { Score } from '@src/model/Score';
 import { Track } from '@src/model/Track';
@@ -330,8 +329,7 @@ export class AlphaTabApiBase<TSettings> {
     public tex(tex: string, tracks?: number[]): void {
         try {
             let parser: AlphaTexImporter = new AlphaTexImporter();
-            let data: ByteBuffer = ByteBuffer.fromString(tex);
-            parser.init(data, this.settings);
+            parser.initFromString(tex, this.settings);
             let score: Score = parser.readScore();
             this.renderScore(score, tracks);
         } catch (e) {

diff --git a/src/importer/AlphaTexImporter.ts b/src/importer/AlphaTexImporter.ts
@@ -33,6 +33,9 @@ import { Logger } from '@src/Logger';
 import { ModelUtils, TuningParseResult } from '@src/model/ModelUtils';
 import { AlphaTabError, AlphaTabErrorType } from '@src/AlphaTabError';
 import { BeatCloner } from '@src/generated/model/BeatCloner';
+import { IOHelper } from '@src/io/IOHelper';
+import { Settings } from '@src/Settings';
+import { ByteBuffer } from '@src/io/ByteBuffer';
 
 /**
  * A list of terminals recognized by the alphaTex-parser
@@ -107,6 +110,7 @@ export class AlphaTexImporter extends ScoreImporter {
     private _score!: Score;
     private _currentTrack!: Track;
     private _currentStaff!: Staff;
+    private _input: string = "";
     private _ch: number = 0;
     private _curChPos: number = 0;
     private _sy: AlphaTexSymbols = AlphaTexSymbols.No;
@@ -129,8 +133,18 @@ export class AlphaTexImporter extends ScoreImporter {
         return 'AlphaTex';
     }
 
+    public initFromString(tex: string, settings: Settings) {
+        this.data = ByteBuffer.empty();
+        this._input = tex;
+        this.settings = settings;
+    }
+
+
     public readScore(): Score {
         try {
+            if(this.data.length > 0) {
+                this._input = IOHelper.toString(this.data.readAll(), this.settings.importer.encoding);
+            }
             this._allowTuning = true;
             this._lyrics = new Map<number, Lyrics[]>();
             this.createDefaultScore();
@@ -148,7 +162,7 @@ export class AlphaTexImporter extends ScoreImporter {
             this.consolidate();
             this._score.finish(this.settings);
             this._score.rebuildRepeatGroups();
-            for(const [track, lyrics] of this._lyrics) {
+            for (const [track, lyrics] of this._lyrics) {
                 this._score.tracks[track].applyLyrics(lyrics);
             }
             return this._score;
@@ -362,12 +376,10 @@ export class AlphaTexImporter extends ScoreImporter {
      * Reads the next character of the source stream.
      */
     private nextChar(): number {
-        let b: number = this.data.readByte();
-        if (b === -1) {
-            this._ch = 0;
+        if (this._curChPos < this._input.length) {
+            this._ch = this._input.charCodeAt(this._curChPos++)
         } else {
-            this._ch = b;
-            this._curChPos++;
+            this._ch = 0;
         }
         return this._ch;
     }
@@ -888,7 +900,7 @@ export class AlphaTexImporter extends ScoreImporter {
             let syData: string = (this._syData as string).toLowerCase();
             if (syData === 'track') {
                 this._staffHasExplicitTuning = false;
-                this._staffTuningApplied = false;        
+                this._staffTuningApplied = false;
 
                 this._sy = this.newSy();
                 // new track starting? - if no masterbars it's the \track of the initial track.
@@ -911,7 +923,7 @@ export class AlphaTexImporter extends ScoreImporter {
                 if (syData === 'staff') {
                     this._staffHasExplicitTuning = false;
                     this._staffTuningApplied = false;
-    
+
                     this._sy = this.newSy();
                     if (this._currentTrack.staves[0].bars.length > 0) {
                         this._currentTrack.ensureStaveCount(this._currentTrack.staves.length + 1);

diff --git a/src/io/ByteBuffer.ts b/src/io/ByteBuffer.ts
@@ -1,5 +1,6 @@
 import { IReadable } from '@src/io/IReadable';
 import { IWriteable } from '@src/io/IWriteable';
+import { IOHelper } from './IOHelper';
 
 export class ByteBuffer implements IWriteable, IReadable {
     private _buffer!: Uint8Array;
@@ -36,10 +37,7 @@ export class ByteBuffer implements IWriteable, IReadable {
     }
 
     public static fromString(contents: string): ByteBuffer {
-        let byteArray: Uint8Array = new Uint8Array(contents.length);
-        for (let i: number = 0; i < contents.length; i++) {
-            byteArray[i] = contents.charCodeAt(i);
-        }
+        let byteArray: Uint8Array = IOHelper.stringToBytes(contents);
         return ByteBuffer.fromBuffer(byteArray);
     }
 

diff --git a/test/TestPlatform.ts b/test/TestPlatform.ts
@@ -1,15 +1,9 @@
-import { ByteBuffer } from '@src/io/ByteBuffer';
-import { IReadable } from '@src/io/IReadable';
 import { IOHelper } from '@src/io/IOHelper';
 
 /**
  * @partial
  */
 export class TestPlatform {
-    public static createStringReader(tex: string): IReadable {
-        return ByteBuffer.fromString(tex);
-    }
-
     /**
      * @target web
      * @partial

diff --git a/test/audio/AlphaSynth.test.ts b/test/audio/AlphaSynth.test.ts
@@ -18,7 +18,7 @@ describe('AlphaSynthTests', () => {
             ' (0.4 0.3).8 r.8(3.4 3.3).8 r.8(5.4 5.3).4 r.8(3.4 3.3).8 | ' +
             'r.8(0.4 0.3).8(-.3 - .4).2 { d } | ';
         let importer: AlphaTexImporter = new AlphaTexImporter();
-        importer.init(TestPlatform.createStringReader(tex), new Settings());
+        importer.initFromString(tex, new Settings());
         let score: Score = importer.readScore();
         let midi: MidiFile = new MidiFile();
         let gen: MidiFileGenerator = new MidiFileGenerator(score, null, new AlphaSynthMidiFileHandler(midi));

diff --git a/test/audio/MidiFileGenerator.test.ts b/test/audio/MidiFileGenerator.test.ts
@@ -35,7 +35,7 @@ import { MetaDataEvent } from '@src/midi/MetaDataEvent';
 describe('MidiFileGeneratorTest', () => {
     const parseTex: (tex: string) => Score = (tex: string): Score => {
         let importer: AlphaTexImporter = new AlphaTexImporter();
-        importer.init(TestPlatform.createStringReader(tex), new Settings());
+        importer.initFromString(tex, new Settings());
         return importer.readScore();
     };
 

diff --git a/test/audio/MidiPlaybackController.test.ts b/test/audio/MidiPlaybackController.test.ts
@@ -4,7 +4,6 @@ import { Score } from '@src/model/Score';
 import { Settings } from '@src/Settings';
 import { Logger } from '@src/Logger';
 import { GpImporterTestHelper } from '@test/importer/GpImporterTestHelper';
-import { TestPlatform } from '@test/TestPlatform';
 
 describe('MidiPlaybackControllerTest', () => {
     const testRepeat: ((score: Score, expectedIndexes: number[]) => void) = (score: Score, expectedIndexes: number[]): void => {
@@ -60,7 +59,7 @@ describe('MidiPlaybackControllerTest', () => {
         let tex: string =
             '\\ro 1.3 2.3 3.3 4.3 | 5.3 6.3 7.3 8.3 | \\rc 2 1.3 2.3 3.3 4.3 | \\ro \\rc 3 1.3 2.3 3.3 4.3';
         let importer: AlphaTexImporter = new AlphaTexImporter();
-        importer.init(TestPlatform.createStringReader(tex), new Settings());
+        importer.initFromString(tex, new Settings());
         let score: Score = importer.readScore();
         let playedBars: number[] = [];
         let controller: MidiPlaybackController = new MidiPlaybackController(score);

diff --git a/test/exporter/Gp7Exporter.test.ts b/test/exporter/Gp7Exporter.test.ts
@@ -129,7 +129,7 @@ describe('Gp7ExporterTest', () => {
         `;
 
         const importer = new AlphaTexImporter();
-        importer.init(TestPlatform.createStringReader(tex), new Settings());
+        importer.initFromString(tex, new Settings());
         const expected = importer.readScore();
         const exported = exportGp7(expected);
 

diff --git a/test/importer/AlphaTexImporter.test.ts b/test/importer/AlphaTexImporter.test.ts
@@ -19,12 +19,11 @@ import { Tuning } from '@src/model/Tuning';
 import { HarmonicsEffectInfo } from '@src/rendering/effects/HarmonicsEffectInfo';
 import { ScoreRenderer } from '@src/rendering/ScoreRenderer';
 import { Settings } from '@src/Settings';
-import { TestPlatform } from '@test/TestPlatform';
 
 describe('AlphaTexImporterTest', () => {
-    const parseTex: (tex:string) => Score = (tex: string): Score => {
+    const parseTex: (tex: string) => Score = (tex: string): Score => {
         let importer: AlphaTexImporter = new AlphaTexImporter();
-        importer.init(TestPlatform.createStringReader(tex), new Settings());
+        importer.initFromString(tex, new Settings());
         return importer.readScore();
     };
 
@@ -856,19 +855,19 @@ describe('AlphaTexImporterTest', () => {
         try {
             parseTex('<xml>');
             fail('Expected error');
-        } catch(e) {
-            if(!(e instanceof UnsupportedFormatError)) {
+        } catch (e) {
+            if (!(e instanceof UnsupportedFormatError)) {
                 fail(`Expected UnsupportedFormatError got ${e}`);
             }
         }
     });
-    
+
     it('expect-invalid-format-other-text', () => {
         try {
             parseTex('This is not an alphaTex file');
             fail('Expected error');
-        } catch(e) {
-            if(!(e instanceof UnsupportedFormatError)) {
+        } catch (e) {
+            if (!(e instanceof UnsupportedFormatError)) {
                 fail(`Expected UnsupportedFormatError got ${e}`);
             }
         }
@@ -891,4 +890,17 @@ describe('AlphaTexImporterTest', () => {
         expect(score.tracks[0].staves[0].tuning.length).toEqual(0);
         expect(score.tracks[0].staves[0].displayTranspositionPitch).toEqual(0);
     });
+
+    it('multibyte-encoding', () => {
+        const multiByteChars = '爱你ÖÄÜ🎸🎵🎶';
+        const score = parseTex(`\\title "${multiByteChars}"
+        .
+        \\track "🎸"
+        \\lyrics "Test Lyrics 🤘"
+        (1.2 1.1).4 x.2.8 0.1 1.1 | 1.2 3.2 0.1 1.1`);
+
+        expect(score.title).toEqual(multiByteChars);
+        expect(score.tracks[0].name).toEqual("🎸");
+        expect(score.tracks[0].staves[0].bars[0].voices[0].beats[2].lyrics![0]).toEqual("🤘");
+    });
 });