Skip to content

Commit

Permalink
Merge pull request #9 from rerender2021/feature/use-gpu
Browse files Browse the repository at this point in the history 
feature: use gpu
  • Loading branch information
@rerender2021
rerender2021 authored Apr 4, 2023
2 parents 0b7801c + 39c9926 commit f6e99d7
Show file tree
Hide file tree
Showing 15 changed files with 510 additions and 108 deletions.
8 changes: 7 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,11 @@ dist
/*.traineddata

asr-server
asr-server-*
nlp-server
/config.json
nlp-gpu-server
/config.json
subtitle
perf
log
*.exe
27 changes: 27 additions & 0 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"dependencies": {
"ave-react": "^0.1.4",
"axios": "^1.3.2",
"debounce": "^1.2.1"
"debounce": "^1.2.1",
"sentence-splitter": "^4.2.0"
}
}
114 changes: 93 additions & 21 deletions src/app.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import React, { useCallback, useEffect, useMemo } from "react";
import { AveRenderer, Grid, Window, getAppContext, IIconResource, IWindowComponentProps, Button, CheckBox, ICheckBoxComponentProps } from "ave-react";
import React, { useCallback, useEffect, useMemo, useState } from "react";
import { AveRenderer, Grid, Window, getAppContext, IIconResource, IWindowComponentProps, Button, CheckBox, ICheckBoxComponentProps, ScrollBar, Label, IScrollBarComponentProps } from "ave-react";
import { App, ThemePredefined_Dark, CheckValue } from "ave-ui";
import { VoskAsrEngine } from "./asr";
import { HelsinkiNlpEngine } from "./nlp";
import { containerLayout, controlLayout } from "./layout";
import { iconResource } from "./resource";
import { onMeasure, onTranslate, shadowRelated } from "./shadow";
import { onMeasure, onTranslate, safe, shadowRelated } from "./shadow";
import { getAsrConfig, getNlpConfig } from "./config";
import axios from "axios";

function onInit(app: App) {
const context = getAppContext();
Expand All @@ -20,6 +21,15 @@ function initTheme() {
themeDark.SetStyle(themeImage, 0);
}

enum ButtonText {
Measure = "设置字幕区",
Recognize = "语音识别",
BreakLongText = "长句分解",
SetTopMost = "字幕置顶",
SubtitleEn = "英文字幕",
SubtitleZh = "中文字幕",
}

export function Echo() {
const asrEngine = useMemo(
() =>
Expand Down Expand Up @@ -57,54 +67,116 @@ export function Echo() {
}, []);

const onSetRecognize = useCallback<ICheckBoxComponentProps["onCheck"]>((sender) => {
shadowRelated.subtitleQueue = [];

let shouldRecognize = false;

const checkValue = sender.GetValue();
if (checkValue === CheckValue.Unchecked) {
shouldRecognize = false;
shadowRelated.onUpdateTranslationResult({ en: "", zh: "" });
} else if (checkValue === CheckValue.Checked) {
shouldRecognize = true;
}

shadowRelated.shouldRecognize = shouldRecognize;
}, []);

const onSetPunct = useCallback<ICheckBoxComponentProps["onCheck"]>((sender) => {
let shouldResotrePunct = false;
const onSetBreakLongText = useCallback<ICheckBoxComponentProps["onCheck"]>((sender) => {
let value = false;

const checkValue = sender.GetValue();
if (checkValue === CheckValue.Unchecked) {
shouldResotrePunct = false;
value = false;
} else if (checkValue === CheckValue.Checked) {
shouldResotrePunct = true;
value = true;
}

shadowRelated.shouldResotrePunct = shouldResotrePunct;
shadowRelated.shouldBreakLongText = value;
}, []);

const onSetDisplaySubtitle = useCallback<ICheckBoxComponentProps["onCheck"]>((sender) => {
const checkValue = sender.GetValue();
const text = sender.GetText();
const isChecked = checkValue === CheckValue.Checked;
if (text === ButtonText.SubtitleEn) {
shadowRelated.subtitleConfig.en = isChecked;
} else if (text === ButtonText.SubtitleZh) {
shadowRelated.subtitleConfig.zh = isChecked;
}
shadowRelated.onUpdateTranslationConfig();
}, []);

const [fontSize, setFontSize] = useState(16);
const onSetFontSize = useCallback<IScrollBarComponentProps["onScrolling"]>((sender) => {
const fontSize = sender.GetValue();
shadowRelated.onUpdateFontSize(fontSize);
setFontSize(fontSize);
}, []);

const [title, setTitle] = useState("Echo");
const [asrReady, setAsrReady] = useState(false);

useEffect(() => {
initTheme();
asrEngine.init();
nlpEngine.init();
asrEngine.init().then(
safe(() => {
setAsrReady(true);
})
);
nlpEngine.init().then(
safe(async () => {
const response = await axios.get("http://localhost:8100/gpu");
if (response.data.gpu === "True") {
console.log("great! use gpu");
setTitle("Echo (GPU)");
} else {
console.log("gpu is not available");
}
})
);
onTranslate(asrEngine, nlpEngine);
}, []);

return (
<Window title="Echo" size={{ width: 260, height: 350 }} onInit={onInit} onClose={onClose}>
<Window title={title} size={{ width: 260, height: 350 }} onInit={onInit} onClose={onClose}>
<Grid style={{ layout: containerLayout }}>
<Grid style={{ area: containerLayout.areas.control, layout: controlLayout }}>
<Grid style={{ area: controlLayout.areas.measure }}>
<Button text="设置字幕区" iconInfo={{ name: "measure", size: 16 }} onClick={onMeasure}></Button>
</Grid>
<Grid style={{ area: controlLayout.areas.recognize }}>
<CheckBox text="语音识别" value={CheckValue.Unchecked} onCheck={onSetRecognize}></CheckBox>
</Grid>
<Grid style={{ area: controlLayout.areas.punct }}>
<CheckBox text="标点恢复" value={CheckValue.Unchecked} onCheck={onSetPunct}></CheckBox>
</Grid>
<Grid style={{ area: controlLayout.areas.topmost }}>
<CheckBox text="字幕置顶" value={CheckValue.Checked} onCheck={onSetTopMost}></CheckBox>
<Button text={ButtonText.Measure} iconInfo={{ name: "measure", size: 16 }} onClick={onMeasure}></Button>
</Grid>
{asrReady ? (
<>
<Grid style={{ area: controlLayout.areas.recognize }}>
<CheckBox text={ButtonText.Recognize} value={CheckValue.Unchecked} onCheck={onSetRecognize}></CheckBox>
</Grid>
<Grid style={{ area: controlLayout.areas.breakLongText }}>
<CheckBox text={ButtonText.BreakLongText} value={CheckValue.Unchecked} onCheck={onSetBreakLongText}></CheckBox>
</Grid>
<Grid style={{ area: controlLayout.areas.topmost }}>
<CheckBox text={ButtonText.SetTopMost} value={CheckValue.Checked} onCheck={onSetTopMost}></CheckBox>
</Grid>
<Grid style={{ area: controlLayout.areas.en }}>
<CheckBox text={ButtonText.SubtitleEn} value={CheckValue.Checked} onCheck={onSetDisplaySubtitle}></CheckBox>
</Grid>
<Grid style={{ area: controlLayout.areas.zh }}>
<CheckBox text={ButtonText.SubtitleZh} value={CheckValue.Checked} onCheck={onSetDisplaySubtitle}></CheckBox>
</Grid>
<Grid style={{ area: controlLayout.areas.fontSizeLabel }}>
<Label text="字体大小"></Label>
</Grid>
<Grid style={{ area: controlLayout.areas.fontSize, margin: "10dpx 0 10dpx 0" }}>
<ScrollBar min={10} max={50} value={16} /** default value */ onScrolling={onSetFontSize}></ScrollBar>
</Grid>
<Grid style={{ area: controlLayout.areas.fontSizeValue }}>
<Label text={`${fontSize}`}></Label>
</Grid>
</>
) : (
<Grid style={{ area: controlLayout.areas.recognize }}>
<Label text="初始化中..."></Label>
</Grid>
)}
</Grid>
</Grid>
</Window>
Expand Down
88 changes: 66 additions & 22 deletions src/asr/asr.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,30 +2,54 @@ import axios from "axios";
import path from "path";
import fs from "fs";
import childProcess from "child_process";
import { IAsrEngine, IAsrEngineOptions, IAsrResult } from "./base";
import { shadowRelated } from "../shadow";
import { IAsrEngine, IAsrEngineOptions, ISentence } from "./base";
import { emptySentence, shadowRelated } from "../shadow";
import { postasr } from "./postasr";

enum AsrVersion {
v100,
v110,
}

export class VoskAsrEngine implements IAsrEngine {
private options: IAsrEngineOptions;
private asr: childProcess.ChildProcessWithoutNullStreams;
private version: AsrVersion;

constructor(options: IAsrEngineOptions) {
this.options = options;
this.version = AsrVersion.v100;
}

getAsrPath() {
const v110 = path.resolve(process.cwd(), "asr-server-v1.1.0");
if (fs.existsSync(v110)) {
this.version = AsrVersion.v110;
console.log("use asr-server-v1.1.0");
return { asrDir: v110, exePath: path.resolve(v110, "./ASR-API.exe") };
}

const v100 = path.resolve(process.cwd(), "asr-server");
if (fs.existsSync(v100)) {
console.log("use asr-server-v1.0.0");
return { asrDir: v100, exePath: path.resolve(v100, "./ASR-API.exe") };
}

return { asrDir: "", exePath: "" };
}

async init() {
console.log("try to init vosk asr engine");
const asrDir = path.resolve(process.cwd(), "asr-server");
const exePath = path.resolve(asrDir, "./ASR-API.exe");
if (fs.existsSync(asrDir) && fs.existsSync(exePath)) {
const { asrDir, exePath } = this.getAsrPath();
if (asrDir && exePath) {
return new Promise((resolve, reject) => {
console.log("asrDir exists, start asr server", asrDir);

const asr = childProcess.spawn(`./asr-server/ASR-API.exe`, [], { windowsHide: true, detached: false /** hide console */ });
const asr = childProcess.spawn(exePath, [], { windowsHide: true, detached: false /** hide console */ });
this.asr = asr;
asr.stdout.on("data", (data) => {
console.log(`stdout: ${data}`);
if (data.includes("ASR-API has been started")) {
if (data.includes("has been started")) {
console.log("asr server started");
resolve(true);
}
Expand Down Expand Up @@ -53,29 +77,49 @@ export class VoskAsrEngine implements IAsrEngine {
}
}

async recognize(): Promise<IAsrResult> {
// const base64 = buffer.toString("base64");
let text = "";
private async asrApi(): Promise<string> {
if (this.version === AsrVersion.v100) {
const response = await axios.post("http://localhost:8200/asr", {}, { timeout: 2000 });
const result = response?.data?.result;
const data = JSON.parse(result || "{}");
const asrText = data.partial || "";
return asrText;
} else {
const response = await axios.post("http://localhost:8200/asr_queue", {}, { timeout: 1000 });
const result = response?.data?.result;
const data = JSON.parse(result[result.length - 1] || "{}");
const asrText = data.partial || "";
return asrText;
}
}

async getAsrResult(): Promise<string> {
let asrResult = "";
try {
const timeout = this.options?.timeout || 3000;
const response = await axios.post("http://localhost:8200/asr", {}, { timeout });
const data = JSON.parse(response.data.result);
console.log(data);
asrResult = await this.asrApi();
} catch (error) {
console.log(`asr failed: ${error.message}`);
} finally {
return asrResult;
}
}

text = data.partial || data.text || "";
async recognize(): Promise<ISentence> {
let sentence: ISentence = emptySentence;
try {
const asrText = await this.getAsrResult();

if (text && shadowRelated.shouldResotrePunct) {
const withPunctResponse = await axios.post("http://localhost:8200/punct", { text }, { timeout });
if (withPunctResponse.data.text) {
text = withPunctResponse.data.text;
console.log({ text });
}
if (shadowRelated.shouldBreakLongText) {
const result = await postasr(asrText);
sentence = { text: result, asr: asrText };
} else {
sentence = { text: asrText, asr: asrText };
}
} catch (error) {
console.log(`asr failed: ${error.message}`);
this.options?.onError(error.message);
} finally {
return { text };
return sentence;
}
}
}
5 changes: 3 additions & 2 deletions src/asr/base.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
export interface IAsrResult {
export interface ISentence {
text: string;
asr: string
}

export interface IAsrEngineOptions {
Expand All @@ -13,7 +14,7 @@ export interface IAsrEngineConstructor {
}

export interface IAsrEngine {
recognize(): Promise<IAsrResult>;
recognize(): Promise<ISentence>;
init(): void;
destroy(): void;
}
Expand Down
3 changes: 2 additions & 1 deletion src/asr/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
export * from "./asr";
export * from "./asr";
export * from "./base";
Loading

0 comments on commit f6e99d7

Please sign in to comment.