asr cost less than 2 secs at the first time

zhouwg · Mar 11, 2024 · d39eb0a · d39eb0a
1 parent e3a5ea0
commit d39eb0a
Show file tree

Hide file tree

Showing 20 changed files with 196 additions and 98 deletions.
diff --git a/build/envsetup.sh b/build/envsetup.sh
@@ -46,7 +46,25 @@ export LOCAL_WHISPERCPP_PATH=${PROJECT_ROOT_PATH}/external/whispercpp
 #modify following lines to adapt to local dev envs
 #export KANTV_TOOLCHAIN_PATH=${PROJECT_ROOT_PATH}/toolchain
 export KANTV_TOOLCHAIN_PATH=/opt/kantv-toolchain
-export ANDROID_NDK=${KANTV_TOOLCHAIN_PATH}/android-ndk-r21e
+#API21：Android 5.0 (Android L)Lollipop
+#API22：Android 5.1 (Android L)Lollipop
+#API23：Android 6.0 (Android M)Marshmallow
+#API24: Android 7.0 (Android N)Nougat
+#API25: Android 7.1 (Android N)Nougat
+#API26：Android 8.0 (Android O)Oreo
+#API27：Android 8.1 (Android O)Oreo
+#API28：Android 9.0 (Android P)Pie
+#API29：Android 10
+#API30：Android 11
+#API31：Android 12
+#API31：Android 12L
+#API33：Android 13
+#API34：Android 14
+export ANDROID_PLATFORM=android-34
+#export ANDROID_NDK=${KANTV_TOOLCHAIN_PATH}/android-ndk-r18b
+#export ANDROID_NDK=${KANTV_TOOLCHAIN_PATH}/android-ndk-r21e
+#export ANDROID_NDK=${KANTV_TOOLCHAIN_PATH}/android-ndk-r24
+export ANDROID_NDK=${KANTV_TOOLCHAIN_PATH}/android-ndk-r26c
 export LOCAL_BAZEL_PATH=${HOME_PATH}/.cache/bazel/_bazel_${BUILD_USER}/d483cd2a2d9204cb5bb4d870c2729238
 export UPSTREAM_WHISPERCPP_PATH=~/cdeos/whisper.cpp
 

diff --git a/build/public.sh b/build/public.sh
@@ -216,14 +216,14 @@ function dump_global_envs()
     echo -e "PROJECT_HOME_PATH:    ${PROJECT_HOME_PATH}"
     echo -e "PROJECT_ROOT_PATH:    ${PROJECT_ROOT_PATH}"
     echo -e "KANTV_TOOLCHAIN_PATH: ${KANTV_TOOLCHAIN_PATH}"
-
     echo -e "host cpu counts:      ${HOST_CPU_COUNTS}"
 
     if [ "${BUILD_TARGET}" = "android" ]; then
     echo -e "\n"
-    echo -e "ANDROID_NDK:               ${ANDROID_NDK}"
-    echo -e "LOCAL_WHISPERCPP_PATH:     ${LOCAL_WHISPERCPP_PATH}"
-    echo -e "UPSTREAM_WHISPERCPP_PATH:  ${UPSTREAM_WHISPERCPP_PATH}"
+    echo -e "${TEXT_RED}ANDROID_PLATFORM:         ${ANDROID_PLATFORM}${TEXT_RESET}"
+    echo -e "${TEXT_RED}ANDROID_NDK:              ${ANDROID_NDK}${TEXT_RESET}"
+    echo -e "LOCAL_WHISPERCPP_PATH:        ${LOCAL_WHISPERCPP_PATH}"
+    echo -e "UPSTREAM_WHISPERCPP_PATH:     ${UPSTREAM_WHISPERCPP_PATH}"
     echo -e "\n"
     fi
 

diff --git a/cdeosplayer/build.gradle b/cdeosplayer/build.gradle
@@ -8,7 +8,7 @@ buildscript {
         google()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:7.0.2'
+        classpath 'com.android.tools.build:gradle:7.0.3'
         classpath 'com.github.dcendents:android-maven-gradle-plugin:1.4.1'
         classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.7'
     }
@@ -30,30 +30,5 @@ task clean(type: Delete) {
 }
 
 wrapper {
-    gradleVersion = '7.0.2'
+    gradleVersion = '7.0.3'
 }
-
-
-
-/*
-API14: Android 4.0-4.0.2 – Ice Cream Sandwich, released October 2011. Build version
-API15：Android 4.0.3 - 4.0.4 Ice Cream Sandwich
-API16：Android 4.1 Jelly Bean
-API17：Android 4.2 Jelly Bean
-API18：Android 4.3 Jelly Bean
-API19：Android 4.4 KitKat
-API20：Android 4.4W Kitkat Watch
-API21：Android 5.0 (Android L)Lollipop
-API22：Android 5.1 (Android L)Lollipop
-API23：Android 6.0 (Android M)Marshmallow
-API24: Android 7.0 (Android N)Nougat
-API25: Android 7.1 (Android N)Nougat
-API26：Android 8.0 (Android O)Oreo
-API27：Android 8.1 (Android O)Oreo
-API28：Android 9.0 (Android P)Pie
-API29：Android 10.0 (Android Q)
-API30：Android 11.0 (Android R)
-API31：Android 12.0
-API32：Android 13.0
-API33：Android 14.0
-*/
diff --git a/cdeosplayer/cdeosplayer-lib/src/main/java/org/ggml/whispercpp/whispercpp.java b/cdeosplayer/cdeosplayer-lib/src/main/java/org/ggml/whispercpp/whispercpp.java
@@ -8,13 +8,15 @@ public class whispercpp {
 
   public static native String get_systeminfo();
 
+  public static native int    get_cpu_core_counts();
+
   public static native void   set_benchmark_status(int bExitBenchmark);
 
   /**
    *
    * @param modelPath
    * @param audioFile
-   * @param nBenchType  0: memcpy 1: mulmat 2: asr 3: full/whisper_encoder
+   * @param nBenchType  0: asr 1: memcpy 2: mulmat  3: full/whisper_encode
    * @param nThreadCounts
    * @return
    */

diff --git a/cdeosplayer/constants.gradle b/cdeosplayer/constants.gradle
@@ -1,4 +1,4 @@
-// Copyright 2017 The Android Open Source Project
+// Copyright 2024 The Android Open Source Project
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,14 +11,36 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+/*
+API14: Android 4.0-4.0.2 – Ice Cream Sandwich, released October 2011. Build version
+API15：Android 4.0.3 - 4.0.4 Ice Cream Sandwich
+API16：Android 4.1 Jelly Bean
+API17：Android 4.2 Jelly Bean
+API18：Android 4.3 Jelly Bean
+API19：Android 4.4 KitKat
+API20：Android 4.4W Kitkat Watch
+API21：Android 5.0 (Android L)Lollipop
+API22：Android 5.1 (Android L)Lollipop
+API23：Android 6.0 (Android M)Marshmallow
+API24: Android 7.0 (Android N)Nougat
+API25: Android 7.1 (Android N)Nougat
+API26：Android 8.0 (Android O)Oreo
+API27：Android 8.1 (Android O)Oreo
+API28：Android 9.0 (Android P)Pie
+API29：Android 10
+API30：Android 11
+API31：Android 12
+API32: Android 12L
+API33：Android 13
+API34：Android 14
+*/
 project.ext {
-    compileSdkVersion = 30
-    buildToolsVersion = "30.0.3"
-    targetSdkVersion  = 28
-    minSdkVersion     = 19
-    //minSdkVersion   = 16
+    //weiguo:2024-03-10, attention here,only works for Xiaomi 14
+    compileSdkVersion = 34
+    buildToolsVersion = "34.0.0"
+    targetSdkVersion  = 34
+    minSdkVersion     = 34
     appTargetSdkVersion = 29
-    ndkVersion = "21.4.7075529"
 
     //version and version code of KanTV
     releaseVersion     = '1.3.0'
@@ -44,4 +66,4 @@ project.ext {
     if (gradle.ext.has('exoplayerModulePrefix')) {
         modulePrefix += gradle.ext.exoplayerModulePrefix
     }
-}
+}
diff --git a/cdeosplayer/kantv/build.gradle b/cdeosplayer/kantv/build.gradle
@@ -44,7 +44,9 @@ android {
         }
     }
 
-    ndkVersion '21.1.6352462'
+    //weiguo:2024-03-10, attention here
+    //ndkVersion '21.1.6352462'
+    ndkVersion  '26.2.11394342'
 
     sourceSets {
         main {

diff --git a/cdeosplayer/kantv/src/main/assets/ggml-tiny.bin b/cdeosplayer/kantv/src/main/assets/ggml-tiny.bin
diff --git a/cdeosplayer/kantv/src/main/java/com/cdeos/kantv/ui/activities/MainActivity.java b/cdeosplayer/kantv/src/main/java/com/cdeos/kantv/ui/activities/MainActivity.java
@@ -267,7 +267,9 @@ protected void onCreate(@Nullable Bundle savedInstanceState) {
 
         CDEUtils.dumpDeviceInfo();
 
-        if (CDEUtils.isEmulator(this)) {
+        //if (CDEUtils.isEmulator(this))
+        if (false)
+        {
             showWarningDialog(this, "can't running this app in emulator");
             navigationView.getMenu().findItem(R.id.navigation_play).setEnabled(false);
             navigationView.getMenu().findItem(R.id.navigation_home).setEnabled(false);

diff --git a/cdeosplayer/kantv/src/main/java/com/cdeos/kantv/ui/fragment/ASRFragment.java b/cdeosplayer/kantv/src/main/java/com/cdeos/kantv/ui/fragment/ASRFragment.java
@@ -89,25 +89,26 @@ public class ASRFragment extends BaseMvpFragment<ASRPresenter> implements ASRVie
 
      Button _btnBenchmark;
 
-     private static final int BECHMARK_MEMCPY       = 0;
-     private static final int BECHMARK_MULMAT       = 1;
-     private static final int BECHMARK_ASR          = 2;
-     private static final int BECHMARK_FULL         = 3; //very slow on Android phone
-
-     private int nThreadCounts                      = 1;
-     private int benchmarkIndex                     = 0;
-     private String strModeName                     = "tiny";
-
-     private long beginTime                         = 0;
-     private long endTime                           = 0;
-     private long duration                          = 0;
+     //keep sync with kantv_asr.h
+     private static final int BECHMARK_ASR = 0;
+     private static final int BECHMARK_MEMCPY = 1;
+     private static final int BECHMARK_MULMAT = 2;
+     private static final int BECHMARK_FULL = 3; //very slow on Android phone
+
+     private int nThreadCounts = 1;
+     private int benchmarkIndex = 0;
+     private String strModeName = "tiny";
+
+     private long beginTime = 0;
+     private long endTime = 0;
+     private long duration = 0;
      private String strBenchmarkInfo;
 
      private AtomicBoolean isBenchmarking = new AtomicBoolean(false);
      private ProgressDialog mProgressDialog;
 
-     private String ggmlModelFileName               = "ggml-tiny.bin";
-     private String ggmlSampleFileName              = "jfk.wav";
+     private String ggmlModelFileName = "ggml-tiny.bin";
+     private String ggmlSampleFileName = "jfk.wav";
 
      private Context mContext;
      private Activity mActivity;
@@ -160,7 +161,13 @@ public void initView() {
          _txtASRInfo.setCompoundDrawablesWithIntrinsicBounds(null, null, null, null);
          displayFileStatus(CDEUtils.getDataPath() + ggmlSampleFileName, CDEUtils.getDataPath() + ggmlModelFileName);
 
-         CDELibraryLoader.load("whispercpp");
+         try {
+             CDELibraryLoader.load("whispercpp");
+             CDELog.j(TAG, "cpu core counts:" + whispercpp.get_cpu_core_counts());
+         } catch (Exception e) {
+             CDELog.j(TAG, "failed to initialize whispercpp jni");
+             return;
+         }
 
          try {
              initKANTVMgr();
@@ -216,6 +223,24 @@ public void onNothingSelected(AdapterView<?> parent) {
              }
          });
 
+     /*
+        <item>tiny</item>
+        <item>tiny.en</item>       good
+        <item>tiny.en-q5_1</item>  good
+        <item>tiny.en-q8_0</item>
+        <item>tiny-q5_1</item>
+        <item>base</item>
+        <item>base.en</item>
+        <item>base-q5_1</item>
+        <item>small</item>
+        <item>small.en</item>
+        <item>small.en-q5_1</item>
+        <item>small-q5_1</item>
+        <item>medium</item>
+        <item>medium.en</item>
+        <item>medium.en-q5_0</item>
+        <item>large</item>
+     */
          Spinner spinnerModelName = mActivity.findViewById(R.id.spinnerModelName);
          String[] arrayModelName = getResources().getStringArray(R.array.modelName);
          ArrayAdapter<String> adapterModel = new ArrayAdapter<String>(mActivity, android.R.layout.simple_spinner_dropdown_item, arrayModelName);
@@ -291,7 +316,6 @@ public void run() {
                  whispercpp.set_benchmark_status(0);
 
 
-
                  while (isBenchmarking.get()) {
                      beginTime = System.currentTimeMillis();
                      strBenchmarkInfo = whispercpp.bench(
@@ -318,7 +342,7 @@ public void run() {
 
 
                              if (strBenchmarkInfo.startsWith("asr_result")) { //when got asr result, playback the audio file
-                                 playAudioFile();
+                                 //playAudioFile();
                              }
 
                              CDELog.j(TAG, benchmarkTip);

diff --git a/cdeosplayer/kantv/src/main/res/values/arrays.xml b/cdeosplayer/kantv/src/main/res/values/arrays.xml
@@ -73,32 +73,39 @@
     </string-array>
 
     <string-array name="benchType">
+        <item>asr</item>
         <item>mempcpy</item>
         <item>mulmat</item>
-        <item>asr</item>
         <item>full</item>
     </string-array>
 
     <string-array name="threadCounts">
-        <item>1</item>
-        <item>2</item>
-        <item>3</item>
-        <item>4</item>
-        <item>5</item>
-        <item>6</item>
-        <item>7</item>
         <item>8</item>
+        <item>7</item>
+        <item>6</item>
+        <item>5</item>
+        <item>4</item>
+        <item>3</item>
+        <item>2</item>
+        <item>1</item>
     </string-array>
 
     <string-array name="modelName">
         <item>tiny</item>
         <item>tiny.en</item>
+        <item>tiny.en-q5_1</item>
+        <item>tiny.en-q8_0</item>
+        <item>tiny-q5_1</item>
         <item>base</item>
         <item>base.en</item>
+        <item>base-q5_1</item>
         <item>small</item>
         <item>small.en</item>
+        <item>small.en-q5_1</item>
+        <item>small-q5_1</item>
         <item>medium</item>
         <item>medium.en</item>
+        <item>medium.en-q5_0</item>
         <item>large</item>
     </string-array>
 </resources>
diff --git a/...12628-Optimizing Memory Copy Routines.pdf → ...-AN12628-OptimizingMemoryCopyRoutines.pdf b/...12628-Optimizing Memory Copy Routines.pdf → ...-AN12628-OptimizingMemoryCopyRoutines.pdf
diff --git a/...ex_a78_core_Technical ReferenceManual.pdf → ...m_cortex_a78_TechnicalReferenceManual.pdf b/...ex_a78_core_Technical ReferenceManual.pdf → ...m_cortex_a78_TechnicalReferenceManual.pdf
diff --git a/external/OpenBLAS/CMakeLists.txt b/external/OpenBLAS/CMakeLists.txt
@@ -5,6 +5,7 @@
 cmake_minimum_required(VERSION 2.8.5)
 
 project(OpenBLAS C ASM)
+set(CMAKE_VERBOSE_MAKEFILE on)
 
 set(OpenBLAS_MAJOR_VERSION 0)
 set(OpenBLAS_MINOR_VERSION 3)
@@ -24,6 +25,15 @@ set(NO_FORTRAN 1)
 set(BUILD_WITHOUT_LAPACK 1)
 set(BUILD_TESTING  0)
 
+add_definitions(-DNDEBUG)
+add_definitions(-O3)
+#add_definitions(-funsafe-math-optimizations)
+#add_definitions(-march=armv8.6-a+simd)
+#add_definitions(-mcpu=cortex-a78)
+add_definitions(-march=armv9+simd)
+add_definitions(-mcpu=cortex-x3)
+add_definitions(-mfloat-abi=hard -mfpu=neon-fp-armv8)
+
 option(BUILD_LAPACK_DEPRECATED "When building LAPACK, include also some older, deprecated routines" ON)
 
 option(BUILD_TESTING "Build LAPACK testsuite when building LAPACK" OFF)
@@ -327,7 +337,7 @@ endif()
 if (NOT NO_CBLAS)
   if (NOT ONLY_CBLAS)
   # Broken without fortran on unix
-    add_subdirectory(utest)
+  #add_subdirectory(utest)
 endif()
 endif()
 

diff --git a/external/OpenBLAS/build.sh b/external/OpenBLAS/build.sh
@@ -38,9 +38,10 @@ fi
 
 function build_arm64
 {
-cmake -H. -B./out/arm64-v8a -DPROJECT_ROOT_PATH=${PROJECT_ROOT_PATH} -DTARGET_NAME=${TARGET} -DCMAKE_BUILD_TYPE=${PROJECT_BUILD_TYPE} -DBUILD_TARGET="android" -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-21 -DANDROID_NDK=${ANDROID_NDK}  -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DLOCAL_WHISPERCPP_PATH=${LOCAL_WHISPERCPP_PATH}
+cmake -H. -B./out/arm64-v8a -DPROJECT_ROOT_PATH=${PROJECT_ROOT_PATH} -DTARGET_NAME=${TARGET} -DCMAKE_BUILD_TYPE=${PROJECT_BUILD_TYPE} -DBUILD_TARGET="android" -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=${ANDROID_PLATFORM} -DANDROID_NDK=${ANDROID_NDK}  -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DLOCAL_WHISPERCPP_PATH=${LOCAL_WHISPERCPP_PATH}
 cd ./out/arm64-v8a
-make -j${HOST_CPU_COUNTS} NO_LAPACK=1 NO_FORTRAN=1 BUILD_TESTING=0 BUILD_WITHOUT_LPACK=1
+#make -j${HOST_CPU_COUNTS} NO_LAPACK=1 NO_FORTRAN=1 BUILD_TESTING=0 BUILD_WITHOUT_LPACK=1
+make  -j${HOST_CPU_COUNTS} NO_LAPACK=1 NO_FORTRAN=1 BUILD_TESTING=0 BUILD_WITHOUT_LPACK=1
 
 ls -l lib/lib${TARGET}.so
 ls -lah lib/lib${TARGET}.so
@@ -51,7 +52,7 @@ cd -
 
 function build_armv7a
 {
-cmake -H. -B./out/armeabi-v7a -DPROJECT_ROOT_PATH=${PROJECT_ROOT_PATH} -DTARGET_NAME=${TARGET} -DCMAKE_BUILD_TYPE=${PROJECT_BUILD_TYPE} -DBUILD_TARGET="android" -DANDROID_ABI=armeabi-v7a -DANDROID_PLATFORM=android-21 -DANDROID_NDK=${ANDROID_NDK}  -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DLOCAL_WHISPERCPP_PATH=${LOCAL_WHISPERCPP_PATH}
+cmake -H. -B./out/armeabi-v7a -DPROJECT_ROOT_PATH=${PROJECT_ROOT_PATH} -DTARGET_NAME=${TARGET} -DCMAKE_BUILD_TYPE=${PROJECT_BUILD_TYPE} -DBUILD_TARGET="android" -DANDROID_ABI=armeabi-v7a -DANDROID_PLATFORM=${ANDROID_PLATFORM} -DANDROID_NDK=${ANDROID_NDK}  -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DLOCAL_WHISPERCPP_PATH=${LOCAL_WHISPERCPP_PATH}
 cd ./out/armeabi-v7a
 make