diff --git a/README-qnn.md b/README-qnn.md index 781a1f3ef8c44f..530348c3b9ad89 100644 --- a/README-qnn.md +++ b/README-qnn.md @@ -86,23 +86,44 @@ TBD ## Android -### I. Setup Environment +### 1. Setup Environment Any **mainstream** Android phone based on Qualcomm's mobile SoC should be supported by llama.cpp + QNN. Qualcomm SM8650-AB Snapdragon 8 Gen 3 based Android phone is preferred. -### II. Build llama.cpp + QNN backend +### 2. Build llama.cpp + QNN backend +- download and install QNN SDK from Qualcomm offcial website -Please refer to [project kantv](https://github.com/zhouwg/kantv) +``` + https://qpm.qualcomm.com/#/main/tools/details/qualcomm_ai_engine_direct + https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools -or +``` + the default installation path is /opt/qcom/aistack/qnn/2.20.0.240223/ -using [test-backend-ops.cpp](tests/ggml-qnn) to verify it on Qualcomm mobile SoC based Android phone -### III. Run the inference on Qualcomm mobile SoC based Android phone +- using test-backend-ops.cpp to verify QNN backend on Qualcomm mobile SoC based Android phone +``` + cd tests/ggml-qnn + + ./build-ggml-qnn.sh + +``` + +### 3. Run UT of QNN backend on Qualcomm mobile SoC based Android phone + +``` + + ./run-ggml-qnn.sh + +``` + +### 4. Run the inference on Qualcomm mobile SoC based Android phone + +Pls refer to [project kantv](https://github.com/zhouwg/kantv) firstly. ![504893116](https://github.com/zhouwg/kantv/assets/6889919/51f0b277-eca4-4938-86f5-415dbf5897e7) diff --git a/tests/ggml-qnn/build-ggml-qnn.sh b/tests/ggml-qnn/build-ggml-qnn.sh index 980b7c6b889ae4..0d8744ced96e8e 100755 --- a/tests/ggml-qnn/build-ggml-qnn.sh +++ b/tests/ggml-qnn/build-ggml-qnn.sh @@ -3,7 +3,7 @@ set -e #modify following lines to adapt to local dev envs -PROJECT_ROOT_PATH=~/github/llama.cpp/ +LLAMACPP_ROOT_PATH=~/github/llama.cpp/ #https://qpm.qualcomm.com/#/main/tools/details/qualcomm_ai_engine_direct #https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools QNN_SDK_PATH=/opt/qcom/aistack/qnn/2.20.0.240223/ @@ -61,7 +61,7 @@ function check_and_download_ndk() function build_arm64 { - cmake -H. -B./out/arm64-v8a -DPROJECT_ROOT_PATH=${PROJECT_ROOT_PATH} -DTARGET_NAME=${TARGET} -DCMAKE_BUILD_TYPE=${PROJECT_BUILD_TYPE} -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=${ANDROID_PLATFORM} -DANDROID_NDK=${ANDROID_NDK} -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DQNN_SDK_PATH=${QNN_SDK_PATH} + cmake -H. -B./out/arm64-v8a -DPROJECT_ROOT_PATH=${LLAMACPP_ROOT_PATH} -DTARGET_NAME=${TARGET} -DCMAKE_BUILD_TYPE=${PROJECT_BUILD_TYPE} -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=${ANDROID_PLATFORM} -DANDROID_NDK=${ANDROID_NDK} -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DQNN_SDK_PATH=${QNN_SDK_PATH} cd ./out/arm64-v8a make diff --git a/tests/ggml-qnn/run-ggml-qnn.sh b/tests/ggml-qnn/run-ggml-qnn.sh index ea68aecb4f9424..ff5e2292108c0f 100755 --- a/tests/ggml-qnn/run-ggml-qnn.sh +++ b/tests/ggml-qnn/run-ggml-qnn.sh @@ -7,16 +7,22 @@ GGML_QNN_TEST=ggml-qnn-test REMOTE_PATH=/data/local/tmp/ adb push ${GGML_QNN_TEST} ${REMOTE_PATH} -adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ -adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ -adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ -#the QNN HTP(aka DSP) backend only verified on Xiaomi14(Qualcomm SM8650-AB Snapdragon 8 Gen 3) successfully -adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ -adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ -adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ -adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ -adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ +adb shell ls ${REMOTE_PATH}/libQnnCpu.so +if [ $? -eq 0 ]; then + printf "QNN libs already exist\n" +else + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnSystem.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnCpu.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnGpu.so ${REMOTE_PATH}/ + + #the QNN HTP(aka DSP) backend only verified on Xiaomi14(Qualcomm SM8650-AB Snapdragon 8 Gen 3) successfully + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtp.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpNetRunExtensions.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpPrepare.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/aarch64-android/libQnnHtpV75Stub.so ${REMOTE_PATH}/ + adb push ${QNN_SDK_PATH}/lib/hexagon-v75/unsigned/libQnnHtpV75Skel.so ${REMOTE_PATH}/ +fi adb shell chmod +x /data/local/tmp/${GGML_QNN_TEST} adb shell /data/local/tmp/${GGML_QNN_TEST}