[runtime/xpu] Support the execution of non-streaming parsing on the Kunlun XPU card #1455

Author: panhehe

runtime/core/cmake/xpu.cmake

@@ -0,0 +1,37 @@
+if(NOT WIN32)
+  string(ASCII 27 Esc)
+  set(ColourReset "${Esc}[m")
+  set(ColourBold  "${Esc}[1m")
+  set(Red         "${Esc}[31m")
+  set(Green       "${Esc}[32m")
+  set(Yellow      "${Esc}[33m")
+  set(Blue        "${Esc}[34m")
+  set(Magenta     "${Esc}[35m")
+  set(Cyan        "${Esc}[36m")
+  set(White       "${Esc}[37m")
+  set(BoldRed     "${Esc}[1;31m")
+  set(BoldGreen   "${Esc}[1;32m")
+  set(BoldYellow  "${Esc}[1;33m")
+  set(BoldBlue    "${Esc}[1;34m")
+  set(BoldMagenta "${Esc}[1;35m")
+  set(BoldCyan    "${Esc}[1;36m")
+  set(BoldWhite   "${Esc}[1;37m")
+endif()
+
+if(XPU)
+  set(RUNTIME_KUNLUN_PATH ${CMAKE_CURRENT_SOURCE_DIR})
+  message(STATUS "RUNTIME_KUNLUN_PATH is ${RUNTIME_KUNLUN_PATH} .\n")
+  set(KUNLUN_XPU_PATH ${RUNTIME_KUNLUN_PATH}/xpu)
+  if(NOT DEFINED ENV{XPU_API_PATH})
+    message(FATAL_ERROR "${BoldRed}NO ENV{XPU_API_PATH} in your env. Please set XPU_API_PATH.${ColourReset}\n")
+  else()
+    set(XPU_API_PATH $ENV{XPU_API_PATH})
+    message("set XPU_API_PATH from env_var. Val is $ENV{XPU_API_PATH}.")
+  endif()
+
+  include_directories(${RUNTIME_KUNLUN_PATH} ${KUNLUN_XPU_PATH}/
+                      ${XPU_API_PATH}/output/include ${XPU_API_PATH}/../runtime/include)
+  link_directories(${XPU_API_PATH}/output/so/ ${XPU_API_PATH}/../runtime/output/so/)
+
+  add_definitions(-DUSE_XPU)
+endif()

runtime/core/decoder/CMakeLists.txt

@@ -7,8 +7,8 @@ set(decoder_srcs
   ctc_endpoint.cc
 )
 
-if(NOT TORCH AND NOT ONNX)
-  message(FATAL_ERROR "Please build with TORCH or ONNX!!!")
+if(NOT TORCH AND NOT ONNX AND NOT XPU)
+  message(FATAL_ERROR "Please build with TORCH or ONNX or XPU!!!")
 endif()
 if(TORCH)
   list(APPEND decoder_srcs torch_asr_model.cc)
@@ -18,7 +18,8 @@ if(ONNX)
 endif()
 
 add_library(decoder STATIC ${decoder_srcs})
-target_link_libraries(decoder PUBLIC kaldi-decoder frontend post_processor utils)
+target_link_libraries(decoder PUBLIC kaldi-decoder frontend
+                      post_processor utils)
 
 if(ANDROID)
   target_link_libraries(decoder PUBLIC ${PYTORCH_LIBRARY} ${FBJNI_LIBRARY})
@@ -29,4 +30,7 @@ else()
   if(ONNX)
     target_link_libraries(decoder PUBLIC onnxruntime)
   endif()
+  if(XPU)
+    target_link_libraries(decoder PUBLIC xpu_conformer)
+  endif()
 endif()

runtime/core/decoder/params.h

@@ -13,7 +13,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-
 #ifndef DECODER_PARAMS_H_
 #define DECODER_PARAMS_H_
 
@@ -29,17 +28,24 @@
 #ifdef USE_TORCH
 #include "decoder/torch_asr_model.h"
 #endif
+#ifdef USE_XPU
+#include "xpu/xpu_asr_model.h"
+#endif
 #include "frontend/feature_pipeline.h"
 #include "post_processor/post_processor.h"
 #include "utils/flags.h"
 #include "utils/string.h"
 
 DEFINE_int32(num_threads, 1, "num threads for ASR model");
+DEFINE_int32(device_id, 0, "set XPU DeviceID for ASR model");
 
 // TorchAsrModel flags
 DEFINE_string(model_path, "", "pytorch exported model path");
 // OnnxAsrModel flags
 DEFINE_string(onnx_dir, "", "directory where the onnx model is saved");
+// XPUAsrModel flags
+DEFINE_string(xpu_model_dir, "",
+              "directory where the XPU model and weights is saved");
 
 // FeaturePipelineConfig flags
 DEFINE_int32(num_bins, 80, "num mel bins for fbank feature");
@@ -66,7 +72,8 @@ DEFINE_double(lattice_beam, 10.0, "lattice beam in ctc wfst search");
 DEFINE_double(acoustic_scale, 1.0, "acoustic scale for ctc wfst search");
 DEFINE_double(blank_skip_thresh, 1.0,
               "blank skip thresh for ctc wfst search, 1.0 means no skip");
-DEFINE_double(length_penalty, 0.0, "length penalty ctc wfst search, will not"
+DEFINE_double(length_penalty, 0.0,
+              "length penalty ctc wfst search, will not"
               "apply on self-loop arc, for balancing the del/ins ratio, "
               "suggest set to -3.0");
 DEFINE_int32(nbest, 10, "nbest for ctc wfst or prefix search");
@@ -130,7 +137,7 @@ std::shared_ptr<DecodeResource> InitDecodeResourceFromFlags() {
 #else
     LOG(FATAL) << "Please rebuild with cmake options '-DONNX=ON'.";
 #endif
-  } else {
+  } else if (!FLAGS_model_path.empty()) {
 #ifdef USE_TORCH
     LOG(INFO) << "Reading torch model " << FLAGS_model_path;
     TorchAsrModel::InitEngineThreads(FLAGS_num_threads);
@@ -140,6 +147,19 @@ std::shared_ptr<DecodeResource> InitDecodeResourceFromFlags() {
 #else
     LOG(FATAL) << "Please rebuild with cmake options '-DTORCH=ON'.";
 #endif
+  } else if (!FLAGS_xpu_model_dir.empty()) {
+#ifdef USE_XPU
+    LOG(INFO) << "Reading XPU WeNet model weight from " << FLAGS_xpu_model_dir;
+    auto model = std::make_shared<XPUAsrModel>();
+    model->SetEngineThreads(FLAGS_num_threads);
+    model->SetDeviceId(FLAGS_device_id);
+    model->Read(FLAGS_xpu_model_dir);
+    resource->model = model;
+#else
+    LOG(FATAL) << "Please rebuild with cmake options '-DXPU=ON'.";
+#endif
+  } else {
+    LOG(FATAL) << "Please set ONNX, TORCH or XPU model path!!!";
   }
 
   LOG(INFO) << "Reading unit table " << FLAGS_unit_path;

runtime/kunlun/.gitignore

@@ -0,0 +1,2 @@
+build/
+fc_base/

runtime/kunlun/CMakeLists.txt

@@ -0,0 +1,81 @@
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
+
+project(wenet VERSION 0.1)
+
+option(CXX11_ABI "whether to use CXX11_ABI libtorch" OFF)
+option(FST_HAVE_BIN "whether to build fst binaries" OFF)
+option(BUILD_TESTING "whether to build unit test" OFF)
+option(GRPC "whether to build with gRPC" OFF)
+# TODO(Binbin Zhang): Change websocket to OFF since it depends on boost
+#                     which is a very big library
+option(WEBSOCKET "whether to build with websocket" OFF)
+option(TORCH "whether to build with Torch" OFF)
+option(XPU "whether to build with XPU" ON)
+option(ONNX "whether to build with ONNX" OFF)
+option(GPU "whether to build with GPU" OFF)
+
+set(CMAKE_VERBOSE_MAKEFILE OFF)
+
+include(FetchContent)
+include(ExternalProject)
+set(FETCHCONTENT_QUIET OFF)
+get_filename_component(fc_base "fc_base" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+set(FETCHCONTENT_BASE_DIR ${fc_base})
+
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+
+if(NOT MSVC)
+  # Keep the same with openfst, -fPIC or -fpic
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -pthread -fPIC")
+else()
+  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+  add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/utf-8>")
+endif()
+
+# Include all dependency
+include(libtorch)
+if(ONNX)
+  include(onnx)
+endif()
+include(openfst)
+if(XPU)
+  include(xpu)
+  # compile xpu_conformer.a and conformer_test
+  add_subdirectory(xpu)
+endif()
+include_directories(
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  ${CMAKE_CURRENT_SOURCE_DIR}/kaldi
+)
+
+# Build all libraries
+add_subdirectory(utils)
+if(NOT MSVC)
+  add_dependencies(utils openfst)
+endif()
+add_subdirectory(frontend)
+add_subdirectory(post_processor)
+add_subdirectory(kaldi)  # kaldi: wfst based decoder
+add_subdirectory(decoder)
+add_subdirectory(api)
+
+# Optionally, you can build with websocket
+if(WEBSOCKET)
+  include(boost)
+  add_subdirectory(websocket)
+endif()
+
+# Optionally, you can build with gRPC
+if(GRPC)
+  include(grpc)
+  add_subdirectory(grpc)
+endif()
+
+# Build all bins
+add_subdirectory(bin)
+
+# Unit Test
+if(BUILD_TESTING)
+  include(gtest)
+  add_subdirectory(test)
+endif()

runtime/kunlun/README.md

@@ -0,0 +1,48 @@
+# 在昆仑芯片上运行Wenet
+## 介绍
+下面的示例展示了如何在XPU上部署WeNet离线或在线的ASR模型。XPU是一种由昆仑芯100%自主研发的通用人工智能计算核心架构。
+
+## 准备XPU运行环境
+
+在开始之前，请确认您获得以下必须的环境。
+
+    XRE(XPU Runtime Environment):昆仑芯片的基础运行环境，包括芯片驱动程序、runtime api库、固件FW工具等功能模块。
+    XDNN(XPU Deep Neural Network Library):加速深度神经网络的昆仑芯片库，提供应用程序中使用的高性能DNN功能库。
+
+如果您需要任何帮助，或是想要进一步了解昆仑芯片，请通过官方网址联系我们：
+https://www.kunlunxin.com.cn/
+
+## 操作步骤
+- 第一步：构建，需要cmake 3.14及以上版本
+
+``` sh
+export CXX=${your_g++_path}
+export CC=${your_gcc_path}
+export XPU_API_PATH=${your_api_path}
+
+# -r : release version; -d : debug version
+bash ./compile.sh -r
+```
+
+- 第二步：测试，测试结果将在控制台输出
+
+``` sh
+## set KUNLUN XPU visible device
+export XPU_VISIBLE_DEVICES=0
+export XPUSIM_DEVICE_MODEL=KUNLUN2
+## set logging level
+export GLOG_logtostderr=1
+export GLOG_v=3
+## set speech wav and model/weight path
+wav_path=${your_test_wav_path}
+xpu_model_dir=${your_xpu_weight_dir}
+units=${your_units.txt}
+## executive command
+./build/bin/decoder_main \
+    --chunk_size -1 \
+    --wav_path ${wav_path} \
+    --xpu_model_dir ${xpu_model_di} \
+    --unit_path ${units}   \
+    --device_id 0           \
+    --nbest  3  2>&1 | tee log.txt
+```

runtime/kunlun/README_EN.md

@@ -0,0 +1,52 @@
+# WeNet running on KUNLUNXIN XPU device
+## Introduction
+The below example shows how to deploy WeNet offline and online ASR models on XPUs.
+XPU is a core architecture 100% independently developed by KUNLUNXIN for general artificial intelligence computing.
+
+## Setup environment for XPU device
+
+Before the start, makesure you have these necessary environment
+
+    XRE(XPU Runtime Environment):The basic operating environment of the XPUs
+    includes functional modules such as chip drivers, runtime api library, and firmware tools.
+
+    XDNN(XPU Deep Neural Network Library): XPU library for accelerating deep neural networks, providing high-performance DNN function library used in applications.
+
+If you would like to know more about XPUs or need any help, please contact us through the official website:
+
+https://www.kunlunxin.com.cn/
+
+## Instruction
+- Step 1. Build, the build requires cmake 3.14 or above.
+
+``` sh
+export CXX=${your_g++_path}
+export CC=${your_gcc_path}
+export XPU_API_PATH=${your_api_path}
+
+# -r : release version; -d : debug version
+bash ./compile.sh -r
+```
+
+- Step 2. Testing, the result is shown in the console.
+
+``` sh
+## set KUNLUN XPU visible device
+export XPU_VISIBLE_DEVICES=0
+export XPUSIM_DEVICE_MODEL=KUNLUN2
+## set logging level
+export GLOG_logtostderr=1
+export GLOG_v=3
+## set speech wav and model/weight/units path
+wav_path=${your_test_wav_path}
+xpu_model_dir=${your_xpu_weight_dir}
+units=${your_units.txt}
+## executive command
+./build/bin/decoder_main \
+    --chunk_size -1 \
+    --wav_path $wav_path \
+    --xpu_model_dir $xpu_model_dir \
+    --unit_path $units   \
+    --device_id 0           \
+    --nbest  3  2>&1 | tee log.txt
+```

runtime/kunlun/api

@@ -0,0 +1 @@
+../core/api

runtime/kunlun/bin

@@ -0,0 +1 @@
+../core/bin

runtime/kunlun/cmake

@@ -0,0 +1 @@
+../core/cmake

runtime/kunlun/compile.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+set -e
+
+usage() {
+    echo "Usage:"
+    echo "bash compile.sh [-r] [-d] [-c]"
+    echo "Description:"
+    echo "-r, build release."
+    echo "-d, build debug."
+    echo "-c, remove cmakecache or build dir, then build."
+    echo "Example 1:"
+    echo "  ./compile.sh -r "
+    echo "  means: remove cache files in build dir, then build release."
+    echo "Example 2:"
+    echo "  ./compile.sh -d -c all "
+    echo "  means: remove all files in build dir, then build debug."
+    exit -1
+}
+
+if [ -z $CXX ]; then
+  echo -e "\033[31m [WARNING]: NO CXX in your env. Suggest setting CXX variable to support C++14. \033[0m"
+  sleep 2
+fi
+
+build_type='Release'
+clean_type='cache'
+
+while getopts 'rdc:h' OPT; do
+    case $OPT in
+        r) build_type="Release";;
+        d) build_type="Debug";;
+        c) clean_type="$OPTARG";;
+        h) usage;;
+        ?) usage;;
+    esac
+done
+
+if [ ! -d ./build ];then
+  mkdir build
+fi
+
+if [ "$clean_type" = "all" ];then
+  pushd build
+  rm -rf ./*
+  popd
+else
+  pushd build
+  rm -rf CMakeFiles/ cmake_install.cmake CMakeCache.txt CPackSourceConfig.cmake
+  popd
+fi
+
+build_cmd="cd build && cmake -DINTTYPES_FORMAT:STRING=C99 "
+
+if [ "$build_type" = "Release" ];then
+  build_cmd="${build_cmd} -DCMAKE_BUILD_TYPE=Release .. && cmake --build ./ "
+else
+  build_cmd="${build_cmd} -DCMAKE_BUILD_TYPE=Debug .. && cmake --build ./ "
+fi
+
+echo "build command is ${build_cmd}"
+
+eval ${build_cmd}