Add ElevenLabs V2 model support and Chinese support (#347)

* Add ElevenLabs V2 model support and Chinese support * Refine env var
Shaunwei · Aug 14, 2023 · 020f765 · 020f765
1 parent 83df08b
commit 020f765
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 3 deletions.
diff --git a/.env.example b/.env.example
@@ -36,6 +36,7 @@ OPEN_AI_WHISPER_API_KEY=YOUR_API_KEY
 # "ELEVEN_LABS" or "GOOGLE_TTS" or "UNREAL_SPEECH"
 TEXT_TO_SPEECH_USE=ELEVEN_LABS
 ELEVEN_LABS_API_KEY=YOUR_API_KEY
+ELEVEN_LABS_USE_V2= # change to true if you have access to V2 model.
 # Add voice id of your cloned voice. leave empty to use default voices
 ELON_MUSK_VOICE_ID=
 LOKI_VOICE_ID=

diff --git a/client/mobile/ios/rac/rac/Welcome/Settings/SettingsView.swift b/client/mobile/ios/rac/rac/Welcome/Settings/SettingsView.swift
@@ -53,7 +53,7 @@ enum LlmOption: RawRepresentable, Hashable, CaseIterable, Identifiable, Codable
 
 enum LanguageOption: RawRepresentable, Hashable, CaseIterable, Identifiable, Codable {
 
-    case english, spanish, french, german, italian, portuguese, polish, hindi
+    case english, spanish, french, german, italian, portuguese, polish, hindi, chinese
 
     init?(rawValue: String) {
         for option in LanguageOption.allCases {
@@ -84,6 +84,8 @@ enum LanguageOption: RawRepresentable, Hashable, CaseIterable, Identifiable, Cod
             return "pl-PL"
         case .hindi:
             return "hi-IN"
+        case .chinese:
+            return "zh-CN"
         }
     }
 
@@ -105,6 +107,8 @@ enum LanguageOption: RawRepresentable, Hashable, CaseIterable, Identifiable, Cod
             return "Polish"
         case .hindi:
             return "Hindi"
+        case .chinese:
+            return "Chinese"
         }
     }
 

diff --git a/client/web/src/components/Languages/index.jsx b/client/web/src/components/Languages/index.jsx
@@ -18,6 +18,7 @@ const Languages = ({ preferredLanguage, setPreferredLanguage }) => {
     'Italian',
     'Polish',
     'Portuguese',
+    'Chinese',
   ];
 
   return (

diff --git a/realtime_ai_character/audio/speech_to_text/whisper.py b/realtime_ai_character/audio/speech_to_text/whisper.py
@@ -31,6 +31,7 @@
     "pt-PT": "pt",
     "hi-IN": "hi",
     "pl-PL": "pl",
+    'zh-CN': 'zh',
 }
 
 

diff --git a/realtime_ai_character/audio/text_to_speech/elevenlabs.py b/realtime_ai_character/audio/text_to_speech/elevenlabs.py
@@ -9,6 +9,9 @@
 
 logger = get_logger(__name__)
 DEBUG = False
+ELEVEN_LABS_MULTILINGUAL_MODEL = 'eleven_multilingual_v2' if os.getenv(
+    "ELEVEN_LABS_USE_V2",
+    'false').lower() in ('true', '1') else 'eleven_multilingual_v1'
 
 config = types.SimpleNamespace(**{
     'chunk_size': 1024,
@@ -71,7 +74,7 @@ async def generate_audio(self, text, voice_id = "", language='en-US') -> bytes:
             voice_id = "21m00Tcm4TlvDq8ikWAM"
         headers = config.headers
         if language != 'en-US':
-            config.data["model_id"] = 'eleven_multilingual_v1'
+            config.data["model_id"] = ELEVEN_LABS_MULTILINGUAL_MODEL
         data = {
             "text": text,
             **config.data,
@@ -83,4 +86,4 @@ async def generate_audio(self, text, voice_id = "", language='en-US') -> bytes:
             if response.status_code != 200:
                 logger.error(f"ElevenLabs returns response {response.status_code}")
             # Get audio/mpeg from the response and return it
-            return response.content
+            return response.content
diff --git a/realtime_ai_character/websocket_routes.py b/realtime_ai_character/websocket_routes.py
@@ -38,6 +38,7 @@
     "pt-PT": "Olá meu amigo, o que te traz aqui hoje?",
     "hi-IN": "नमस्ते मेरे दोस्त, आज आपको यहां क्या लाया है?",
     "pl-PL": "Cześć mój przyjacielu, co cię tu dziś przynosi?",
+    "zh-CN": "嗨，我的朋友，今天你为什么来这里？",
 }