From ff57733e2b1b9b7c327cde8031a5d8b3d2db085e Mon Sep 17 00:00:00 2001 From: quic-zhanweiw <150764245+quic-zhanweiw@users.noreply.github.com> Date: Sun, 7 Jul 2024 09:07:03 +0800 Subject: [PATCH] Delete SvcQNNHelper directory Signed-off-by: quic-zhanweiw <150764245+quic-zhanweiw@users.noreply.github.com> --- SvcQNNHelper/CMakeLists.txt | 27 -- SvcQNNHelper/src/Utils/ShareMem.hpp | 122 --------- SvcQNNHelper/src/Utils/Utils.hpp | 369 -------------------------- SvcQNNHelper/src/main.cpp | 395 ---------------------------- 4 files changed, 913 deletions(-) delete mode 100644 SvcQNNHelper/CMakeLists.txt delete mode 100644 SvcQNNHelper/src/Utils/ShareMem.hpp delete mode 100644 SvcQNNHelper/src/Utils/Utils.hpp delete mode 100644 SvcQNNHelper/src/main.cpp diff --git a/SvcQNNHelper/CMakeLists.txt b/SvcQNNHelper/CMakeLists.txt deleted file mode 100644 index 072e2e7..0000000 --- a/SvcQNNHelper/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -#============================================================================= -# -# Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause -# -#============================================================================= -cmake_minimum_required(VERSION 3.4...3.18) -project(SvcQNNHelper) - -set(APP "SvcQNNHelper") -set(APP_SOURCES "src/main.cpp") - -LINK_DIRECTORIES(../lib/Release ../lib/RelWithDebInfo) - -add_executable(${APP} ${APP_SOURCES}) - -target_link_libraries(${PROJECT_NAME} PUBLIC libqnnhelper) -SET(EXECUTABLE_OUTPUT_PATH "${PROJECT_SOURCE_DIR}/../lib") - -target_compile_definitions(${APP} PUBLIC "-DNOMINMAX") -target_link_libraries(${APP} PRIVATE Shlwapi Shell32) -set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd") -set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MD /O2 /Ob2") -target_include_directories(${APP} PUBLIC CachingUtil - ../LibQNNHelper/src - ./) diff --git a/SvcQNNHelper/src/Utils/ShareMem.hpp b/SvcQNNHelper/src/Utils/ShareMem.hpp deleted file mode 100644 index 0c722b6..0000000 --- a/SvcQNNHelper/src/Utils/ShareMem.hpp +++ /dev/null @@ -1,122 +0,0 @@ -//============================================================================== -// -// Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved. -// -// SPDX-License-Identifier: BSD-3-Clause -// -//============================================================================== - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "LibQNNHelper.hpp" - - -#define PRINT_MEMINFO (0) - -typedef struct ShareMemInfo { - size_t size; - HANDLE hCreateMapFile; - LPVOID lpBase; -} ShareMemInfo_t; - -std::unordered_map sg_share_mem_map; - -BOOL Print_MemInfo(std::string TAG) { -#if PRINT_MEMINFO - /* - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); - if (!GlobalMemoryStatusEx(&memInfo)) { - QNN_ERR("GlobalMemoryStatusEx failed."); - return false; - } - - // std::cout << "Shared memory used by this process: " << memInfo.ullAvailPageFile << " bytes" << std::endl; - QNN_INF("MemInfo:: %d M", memInfo.ullAvailPageFile / 1024 / 1024); - */ - - uint64_t phyUsed = 0, memUsed = 0, pagefileUsed = 0; - PROCESS_MEMORY_COUNTERS_EX pmc; - DWORD processID = GetCurrentProcessId(); - HANDLE processHandle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, processID); - if (GetProcessMemoryInfo(processHandle, (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) { - phyUsed = pmc.WorkingSetSize / 1024 / 1024; - memUsed = pmc.PrivateUsage / 1024 / 1024; - pagefileUsed = pmc.PagefileUsage / 1024 / 1024; - } - CloseHandle(processHandle); - QNN_WAR("[MemInfo][%s]:: phy used: %llu M, mem used: %llu M, pagefile used %llu M", TAG.c_str(), phyUsed, memUsed, pagefileUsed); - -#endif - return true; -} - -ShareMemInfo_t* FindShareMem(std::string share_memory_name) { - auto it = sg_share_mem_map.find(share_memory_name); - if (it != sg_share_mem_map.end()) { - if (it->second) { - auto pShareMemInfo = it->second; - return pShareMemInfo; - } - } - - QNN_ERR("FindShareMem::find failed.\n"); - return nullptr; -} - -BOOL CreateShareMem(std::string share_memory_name, size_t share_memory_size) { - DWORD size = (DWORD)share_memory_size; - HANDLE hCreateMapFile = nullptr; - LPVOID lpBase = nullptr; - ShareMemInfo_t* pShareMemInfo = nullptr; - - hCreateMapFile = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, - size, share_memory_name.c_str()); - - if(hCreateMapFile) { - lpBase = MapViewOfFile(hCreateMapFile, FILE_MAP_ALL_ACCESS, 0, 0, size); - } - - if (lpBase) { - pShareMemInfo = (ShareMemInfo_t*)malloc(sizeof(ShareMemInfo_t)); - if (pShareMemInfo) { - pShareMemInfo->size = share_memory_size; - pShareMemInfo->hCreateMapFile = hCreateMapFile; - pShareMemInfo->lpBase = lpBase; - - sg_share_mem_map.insert(std::make_pair(share_memory_name, pShareMemInfo)); - QNN_INF("CreateShareMem::Count = %d\n", (int)sg_share_mem_map.size()); - return true; - } - } - - QNN_ERR("CreateShareMem::create failed.\n"); - return false; -} - -BOOL DeleteShareMem(std::string share_memory_name) { - ShareMemInfo_t* pShareMemInfo = FindShareMem(share_memory_name); - if (!pShareMemInfo) { - QNN_ERR("DeleteShareMem::Cant find this share memory %s.\n", share_memory_name.c_str()); - return false; - } - else { - UnmapViewOfFile(pShareMemInfo->lpBase); - CloseHandle(pShareMemInfo->hCreateMapFile); - sg_share_mem_map.erase(share_memory_name); - free(pShareMemInfo); - QNN_INF("DeleteShareMem::Count = %d\n", (int)sg_share_mem_map.size()); - return true; - } - - return false; -} diff --git a/SvcQNNHelper/src/Utils/Utils.hpp b/SvcQNNHelper/src/Utils/Utils.hpp deleted file mode 100644 index 98e6388..0000000 --- a/SvcQNNHelper/src/Utils/Utils.hpp +++ /dev/null @@ -1,369 +0,0 @@ -//============================================================================== -// -// Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved. -// -// SPDX-License-Identifier: BSD-3-Clause -// -//============================================================================== - -#pragma once - -#ifndef _LIBQNNHELPER_UTILS_H -#define _LIBQNNHELPER_UTILS_H - - -#include -#include - -#include "Utils/ShareMem.hpp" - -#define GLOBAL_BUFSIZE 4096 - -#ifdef UNICODE -#define SVC_QNNHELPER_CMD TEXT("SvcQNNHelper.exe svc %llu %llu %llu %d %d \"%S\"") -#else -#define SVC_QNNHELPER_CMD TEXT("SvcQNNHelper.exe svc %llu %llu %llu %d %d \"%s\"") -#endif - -uint64_t g_logEpoch = 0; -int g_logLevel = 0; -int g_profilingLevel = 0; -std::string g_ProcName = "^main"; - -char g_buffer[GLOBAL_BUFSIZE]; - -typedef struct ProcInfo { - HANDLE hSvcPipeInWrite; - HANDLE hSvcPipeOutRead; - PROCESS_INFORMATION piSvcProcInfo; -} ProcInfo_t; - -std::unordered_map sg_proc_info_map; // proc_name map to ProcInfo_t. -std::unordered_map sg_model_info_map; // model_name map to ProcInfo_t. - -std::string GetLastErrorAsString(std::string message) { - DWORD errorMessageID = ::GetLastError(); - if (errorMessageID == 0) - return std::string(); - - LPSTR messageBuffer = nullptr; - size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&messageBuffer, 0, NULL); - std::string result(messageBuffer, size); - LocalFree(messageBuffer); - - return message + " Error: [" + std::to_string(errorMessageID) + "] " + result; -} - -void ErrorExit(std::string message) { - QNN_ERR(GetLastErrorAsString(message).c_str()); - ExitProcess(1); -} - -void split_string(std::vector & output, const std::string &input, const char separator) { - std::istringstream tokenStream(input); - while (!tokenStream.eof()) { - std::string value; - getline(tokenStream, value, separator); - if (!value.empty()) { - output.push_back(value); - } - } -} - -ProcInfo_t* FindProcInfo(std::string proc_name) { - auto it = sg_proc_info_map.find(proc_name); - if (it != sg_proc_info_map.end()) { - if (it->second) { - auto pProcInfo = it->second; - return pProcInfo; - } - } - - return nullptr; -} - -ProcInfo_t* CreateSvcProcess(std::string proc_name) { - STARTUPINFO siStartInfo; - PROCESS_INFORMATION piSvcProcInfo; - ProcInfo_t* pProcInfo = nullptr; - BOOL bSuccess = FALSE; - - HANDLE hSvcPipeInRead = NULL; - HANDLE hSvcPipeInWrite = NULL; - HANDLE hSvcPipeOutRead = NULL; - HANDLE hSvcPipeOutWrite = NULL; - - SECURITY_ATTRIBUTES saAttr; - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - saAttr.lpSecurityDescriptor = NULL; - - if (!CreatePipe(&hSvcPipeOutRead, &hSvcPipeOutWrite, &saAttr, 0)) - ErrorExit("Create out pipe failed."); - - if (!SetHandleInformation(hSvcPipeOutRead, HANDLE_FLAG_INHERIT, 0)) - ErrorExit("SetHandleInformation for out pipe failed."); - - if (!CreatePipe(&hSvcPipeInRead, &hSvcPipeInWrite, &saAttr, 0)) - ErrorExit("Create in pipe failed"); - - if (!SetHandleInformation(hSvcPipeInWrite, HANDLE_FLAG_INHERIT, 0)) - ErrorExit("SetHandleInformation for in pipe failed."); - - ZeroMemory(&piSvcProcInfo, sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - - siStartInfo.cb = sizeof(STARTUPINFO); - - _stprintf_s((TCHAR*)g_buffer, GLOBAL_BUFSIZE, SVC_QNNHELPER_CMD, (uint64_t)hSvcPipeInRead, (uint64_t)hSvcPipeOutWrite, - g_logEpoch, g_logLevel, g_profilingLevel, proc_name.c_str()); - - bSuccess = CreateProcess(NULL, (TCHAR*)g_buffer, NULL, NULL, TRUE, 0, NULL, NULL, &siStartInfo, &piSvcProcInfo); - - if (!bSuccess) { - ErrorExit("CreateProcess failed."); - } - else { - CloseHandle(hSvcPipeOutWrite); - CloseHandle(hSvcPipeInRead); - - ProcInfo_t* pProcInfo = (ProcInfo_t*)malloc(sizeof(ProcInfo_t)); - pProcInfo->hSvcPipeOutRead = hSvcPipeOutRead; - pProcInfo->hSvcPipeInWrite = hSvcPipeInWrite; - pProcInfo->piSvcProcInfo = piSvcProcInfo; - - sg_proc_info_map.insert(std::make_pair(proc_name, pProcInfo)); - - QNN_INF("CreateSvcProcess Success!"); - return pProcInfo; - } - - return nullptr; -} - -BOOL StopSvcProcess(std::string proc_name) { - ProcInfo_t* pProcInfo = FindProcInfo(proc_name); - if (!pProcInfo) { - QNN_ERR("TalkToSvc_Inference::Cant find this process %s.\n", proc_name.c_str()); - return false; - } - - CloseHandle(pProcInfo->piSvcProcInfo.hProcess); - CloseHandle(pProcInfo->piSvcProcInfo.hThread); - CloseHandle(pProcInfo->hSvcPipeInWrite); // This will close pipe write for Svc process, it will exit. - CloseHandle(pProcInfo->hSvcPipeOutRead); - sg_proc_info_map.erase(proc_name); - free(pProcInfo); - return true; -} - -// Send model data to the Svc through share meoory and receive model generated data from share memory. -BOOL TalkToSvc_Initialize(const std::string& model_name, const std::string& proc_name, const std::string& model_path, - const std::string& backend_lib_path, const std::string& system_lib_path) { - ProcInfo_t* pProcInfo = FindProcInfo(proc_name); - if (!pProcInfo) { - pProcInfo = CreateSvcProcess(proc_name); - - if (!pProcInfo) return false; - } - - HANDLE hSvcPipeInWrite = pProcInfo->hSvcPipeInWrite; - HANDLE hSvcPipeOutRead = pProcInfo->hSvcPipeOutRead; - DWORD dwRead = 0, dwWrite = 0; - BOOL bSuccess; - - std::string command = "l" + model_name + ";" + model_path + ";" + backend_lib_path + ";" + system_lib_path; - dwRead = (DWORD)command.length() + 1; - - TimerHelper timerHelper; - // Write command to Svc. - bSuccess = WriteFile(hSvcPipeInWrite, command.c_str(), dwRead, &dwWrite, NULL); - // QNN_INF("TalkToSvc_Initialize::WriteToPipe: %s dwRead = %d dwWrite = %d\n", command.c_str(), dwRead, dwWrite); - if (!bSuccess) return false; - - // Read command from Svc. - bSuccess = ReadFile(hSvcPipeOutRead, g_buffer, GLOBAL_BUFSIZE, &dwRead, NULL); - if(dwRead) { - g_buffer[dwRead] = 0; - QNN_INF("TalkToSvc_Initialize::ReadFromPipe: %s dwRead = %d\n", g_buffer, dwRead); - } - else { - QNN_ERR("TalkToSvc_Initialize::ReadFromPipe: Failed to read from hSvcPipeOutRead, perhaps child process died.\n"); - } - if (!bSuccess || dwRead == 0) return false; - timerHelper.Print("TalkToSvc_Initialize::Pipe talk"); - - // Add "model_name" to "sg_model_info_map". - sg_model_info_map.insert(std::make_pair(model_name, pProcInfo)); - - return bSuccess; -} - -BOOL TalkToSvc_Destroy(std::string model_name, std::string proc_name) { - ProcInfo_t* pProcInfo = FindProcInfo(proc_name); - if (!pProcInfo) { - QNN_ERR("TalkToSvc_Destroy::Cant find this process %s.\n", proc_name.c_str()); - return false; - } - - HANDLE hSvcPipeInWrite = pProcInfo->hSvcPipeInWrite; - HANDLE hSvcPipeOutRead = pProcInfo->hSvcPipeOutRead; - DWORD dwRead = 0, dwWrite = 0; - BOOL bSuccess; - - std::string command = "r" + model_name; - dwRead = (DWORD)command.length() + 1; - - TimerHelper timerHelper; - // Write command to Svc. - bSuccess = WriteFile(hSvcPipeInWrite, command.c_str(), dwRead, &dwWrite, NULL); - QNN_INF("TalkToSvc_Destroy::WriteToPipe: %s dwRead = %d dwWrite = %d\n", command.c_str(), dwRead, dwWrite); - if (!bSuccess) return false; - - // Read command from Svc. - bSuccess = ReadFile(hSvcPipeOutRead, g_buffer, GLOBAL_BUFSIZE, &dwRead, NULL); - if (dwRead) { - g_buffer[dwRead] = 0; - QNN_INF("TalkToSvc_Destroy::ReadFromPipe: %s dwRead = %d\n", g_buffer, dwRead); - } - else { - QNN_ERR("TalkToSvc_Destroy::ReadFromPipe: Failed to read from hSvcPipeOutRead, perhaps child process died.\n"); - } - if (!bSuccess || dwRead == 0) return false; - timerHelper.Print("TalkToSvc_Destroy::Pipe talk"); - - sg_model_info_map.erase(model_name); - if(sg_model_info_map.size() == 0) { // If no model in this process, stop this process. - QNN_INF("TalkToSvc_Destroy::StopSvcProcess.\n"); - StopSvcProcess(proc_name); - } - - return bSuccess; -} - -// The format of strStringSize: "124,3333,434343,132", included the inputSize content. -void ShareMemToVector(std::string strBufferArray, uint8_t* lpBase, std::vector& buffers, std::vector& size) { - std::vector strArray; - std::vector strOffsetArray; - std::vector strSizeArray; - split_string(strArray, strBufferArray, '='); - split_string(strOffsetArray, strArray[0], ','); - split_string(strSizeArray, strArray[1], ','); - - size_t offset = 0; - size_t dataSize = 0; - - // Perhaps the data in buffer is not in order. - for (int i = 0; i < strOffsetArray.size(); i++) { - offset = std::stoull(strOffsetArray[i]); - dataSize = std::stoull(strSizeArray[i]); - size.push_back(dataSize); - buffers.push_back(reinterpret_cast(lpBase + offset)); - } -} - -// Copy data to 'pShareMemInfo->lpBase'. If the data in 'buffers' has been in the area of share memory, don't copy. -std::pair VectorToShareMem(size_t share_memory_size, uint8_t* lpBase, std::vector& buffers, std::vector& size) { - QNN_INF("VectorToShareMem Start. size = %llu\n", share_memory_size); - //TimerHelper timerHelper; - - std::string strOffsetArray = ""; - std::string strSizeArray = ""; - size_t offset = 0; - size_t dataSize = 0; - uint8_t* buffer = nullptr; - - // How to handle the case - part of the data in buffers are in the share memory? - // Calculate the offset, avoid overflow the input data which already in the share memory. - for (int i = 0; i < buffers.size(); i++) { - buffer = buffers[i]; - if (buffer >= lpBase && buffer <= lpBase + share_memory_size) { // This buffer is in the share memory area. - offset += size[i]; - } - } - - // Copy the data which is not in share memory to share memory. - for (int i = 0; i < buffers.size(); i++) { - buffer = buffers[i]; - dataSize = size[i]; - if (buffer >= lpBase && buffer <= lpBase + share_memory_size) { // This buffer is in the share memory area. - strOffsetArray += std::to_string(buffer - lpBase) + ","; - //QNN_INF("VectorToShareMem in buffers, ignore copy.\n"); - } - else { - memcpy((uint8_t*)lpBase + offset, buffers[i], dataSize); // This buffer is NOT in the share memory area, copy it. - strOffsetArray += std::to_string(offset) + ","; - offset += dataSize; - //QNN_INF("VectorToShareMem NOT in buffers, copy...\n"); - } - strSizeArray += std::to_string(dataSize) + ","; - } - - //timerHelper.Print("VectorToShareMem::offset = " + std::to_string(offset)); - // QNN_INF("VectorToShareMem End.\n"); - // QNN_INF("VectorToShareMem::strOffsetArray = %s.\n", strOffsetArray.c_str()); - return std::make_pair(strOffsetArray, strSizeArray); -} - -// Send model data to the Svc through share memory and receive model generated data from share memory. -BOOL TalkToSvc_Inference(std::string model_name, std::string proc_name, std::string share_memory_name, - std::vector& inputBuffers, std::vector& inputSize, - std::vector& outputBuffers, std::vector& outputSize, - std::string perfProfile) { - ProcInfo_t* pProcInfo = FindProcInfo(proc_name); - if (!pProcInfo) { - QNN_ERR("TalkToSvc_Inference::Cant find this process %s.\n", proc_name.c_str()); - return false; - } - - ShareMemInfo_t* pShareMemInfo = FindShareMem(share_memory_name); - if (!pShareMemInfo) { - QNN_ERR("TalkToSvc_Inference::Cant find this share memory %s.\n", share_memory_name.c_str()); - return false; - } - - HANDLE hSvcPipeInWrite = pProcInfo->hSvcPipeInWrite; - HANDLE hSvcPipeOutRead = pProcInfo->hSvcPipeOutRead; - DWORD dwRead = 0, dwWrite = 0; - BOOL bSuccess; - - std::string command = "g" + model_name + ";" + share_memory_name + ";" + std::to_string(pShareMemInfo->size) + ";"; - // 'offset' in share memory(according to 'inputBuffers' data size, so that we can restore this data to 'std::vector' in Svc). - std::pair strResultArray = VectorToShareMem(pShareMemInfo->size, (uint8_t*)pShareMemInfo->lpBase, inputBuffers, inputSize); - command = command + strResultArray.first + "=" + strResultArray.second + ";"; - command = command + perfProfile; - dwRead = (DWORD)command.length() + 1; - - // start_time(); - // Write command to Svc. - bSuccess = WriteFile(hSvcPipeInWrite, command.c_str(), dwRead, &dwWrite, NULL); - // QNN_INF("TalkToSvc_Inference::WriteToPipe: %s dwRead = %d dwWrite = %d\n", command.c_str(), dwRead, dwWrite); - if (!bSuccess) return false; - - // Read command from Svc. - bSuccess = ReadFile(hSvcPipeOutRead, g_buffer, GLOBAL_BUFSIZE, &dwRead, NULL); - if(dwRead) { - g_buffer[dwRead] = 0; - QNN_INF("TalkToSvc_Inference::ReadFromPipe: %s dwRead = %d\n", g_buffer, dwRead); - } - else { - QNN_ERR("TalkToSvc_Inference::ReadFromPipe: Failed to read from hSvcPipeOutRead, perhaps child process died.\n"); - } - if (!bSuccess || dwRead == 0) return false; - //print_time("TalkToSvc_Inference::Pipe talk"); - - // Read the output data from 'share_memory_name'. - if (dwRead) { - if (g_buffer[0] == 'F') { // ACTION_FAILED == Failed. - return false; - } - - ShareMemToVector(g_buffer, (uint8_t*)pShareMemInfo->lpBase, outputBuffers, outputSize); - } - - return bSuccess; -} - -#endif - diff --git a/SvcQNNHelper/src/main.cpp b/SvcQNNHelper/src/main.cpp deleted file mode 100644 index b81bd1d..0000000 --- a/SvcQNNHelper/src/main.cpp +++ /dev/null @@ -1,395 +0,0 @@ -//============================================================================== -// -// Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved. -// -// SPDX-License-Identifier: BSD-3-Clause -// -//============================================================================== - -#pragma once - - -#include "Utils/Utils.hpp" - - -// ============================== Service / SvcQNNHelper ============================== // - -#define ACTION_OK "OK" -#define ACTION_FAILED "Failed" - -LibQNNHelper g_LibQNNHelper; - -LPVOID OpenShareMem(std::string share_memory_name, size_t share_memory_size) { - HANDLE hOpenMapFile = nullptr; - LPVOID lpBase = nullptr; - ShareMemInfo_t* pShareMemInfo = nullptr; - - hOpenMapFile = OpenFileMappingA(FILE_MAP_ALL_ACCESS, NULL, share_memory_name.c_str()); - - if (hOpenMapFile) { - lpBase = MapViewOfFile(hOpenMapFile, FILE_MAP_ALL_ACCESS, 0, 0, 0); - } - if (lpBase) { - pShareMemInfo = (ShareMemInfo_t*)malloc(sizeof(ShareMemInfo_t)); - - if (pShareMemInfo) { - pShareMemInfo->hCreateMapFile = hOpenMapFile; - pShareMemInfo->lpBase = lpBase; - pShareMemInfo->size = share_memory_size; - // QNN_INF("OpenShareMem::pShareMemInfo %p\n", pShareMemInfo); - sg_share_mem_map.insert(std::make_pair(share_memory_name, pShareMemInfo)); - } - } - - return lpBase; -} - -void CloseShareMem(std::string share_memory_name) { - ShareMemInfo_t* pShareMemInfo = sg_share_mem_map[share_memory_name]; - - if (pShareMemInfo) { - // QNN_INF("CloseShareMem::pShareMemInfo %p\n", pShareMemInfo); - UnmapViewOfFile(pShareMemInfo->lpBase); - CloseHandle(pShareMemInfo->hCreateMapFile); - sg_share_mem_map.erase(share_memory_name); - free(pShareMemInfo); - } - else { - QNN_ERR("CloseShareMem::Can't find share memory%s.\n", share_memory_name.c_str()); - } -} - -void ModelLoad(std::string cmdBuf, HANDLE hSvcPipeOutWrite) { - BOOL bSuccess; - Print_MemInfo("ModelLoad Start."); - - std::vector commands; - split_string(commands, cmdBuf, ';'); - - std::string model_name = commands[0]; - std::string model_path = commands[1]; - std::string backend_lib_path = commands[2]; - std::string system_lib_path = commands[3]; - - Print_MemInfo("ModelLoad::ModelInitialize Start."); - QNN_INF("ModelLoad::ModelInitialize::Model name %s\n", model_name.c_str()); - bSuccess = g_LibQNNHelper.ModelInitialize(model_name.c_str(), model_path, backend_lib_path, system_lib_path); - QNN_INF("ModelLoad::ModelInitialize End ret = %d\n", bSuccess); - Print_MemInfo("ModelLoad::ModelInitialize End."); - - if(bSuccess) { - bSuccess = WriteFile(hSvcPipeOutWrite, ACTION_OK, (DWORD)strlen(ACTION_OK) + 1, NULL, NULL); - } - else { - bSuccess = WriteFile(hSvcPipeOutWrite, ACTION_FAILED, (DWORD)strlen(ACTION_FAILED) + 1, NULL, NULL); - } -} - -void ModelRun(std::string cmdBuf, HANDLE hSvcPipeOutWrite) { - BOOL bSuccess; - Print_MemInfo("ModelRun Start."); - // TimerHelper timerHelper; - - std::vector commands; - split_string(commands, cmdBuf, ';'); - - std::string model_name = commands[0]; - std::string share_memory_name = commands[1]; - size_t share_memory_size = std::stoull(commands[2]); - std::string strBufferArray = commands[3]; - std::string perfProfile = commands[4]; - - - // Open share memory and read the inference data from share memory. - LPVOID lpBase = OpenShareMem(share_memory_name, share_memory_size); - - std::vector inputBuffers; - std::vector inputSize; - std::vector outputBuffers; - std::vector outputSize; - outputSize.push_back(12345); - - // Fill data from 'pShareMemInfo->lpBase' to 'inputBuffers' vector before inference the model. - ShareMemToVector(strBufferArray, (uint8_t*)lpBase, inputBuffers, inputSize); - - Print_MemInfo("ModelRun::ModelInference Start."); - //QNN_INF("ModelRun::ModelInference %s\n", model_name.c_str()); - bSuccess = g_LibQNNHelper.ModelInference(model_name.c_str(), inputBuffers, outputBuffers, outputSize, perfProfile); - //QNN_INF("ModelRun::ModelInference End ret = %d\n", bSuccess); - Print_MemInfo("ModelRun::ModelInference End."); - - // Fill data from outputBuffers to 'pShareMemInfo->lpBase' and send back to client. - std::pair strResultArray = VectorToShareMem(share_memory_size, (uint8_t*)lpBase, outputBuffers, outputSize); - - outputBuffers.clear(); - outputSize.clear(); - - Print_MemInfo("ModelRun::CloseShareMem Start."); - CloseShareMem(share_memory_name); - Print_MemInfo("ModelRun::CloseShareMem End."); - - // timerHelper.Print("ModelRun"); - - DWORD dwRead = 0, dwWrite = 0; - std::string command = strResultArray.first + "=" + strResultArray.second; - dwRead = (DWORD)command.length() + 1; - if (bSuccess) { - bSuccess = WriteFile(hSvcPipeOutWrite, command.c_str(), dwRead, &dwWrite, NULL); - } - else { - bSuccess = WriteFile(hSvcPipeOutWrite, ACTION_FAILED, (DWORD)strlen(ACTION_FAILED) + 1, NULL, NULL); - } -} - -void ModelRelease(std::string cmdBuf, HANDLE hSvcPipeOutWrite) { - BOOL bSuccess; - Print_MemInfo("ModelRelease Start."); - - std::vector commands; - split_string(commands, cmdBuf, ';'); - - std::string model_name = commands[0]; - - Print_MemInfo("ModelRelease::ModelDestroy Start."); - QNN_INF("ModelRelease::ModelDestroy %s\n", model_name.c_str()); - bSuccess = g_LibQNNHelper.ModelDestroy(model_name.c_str()); - QNN_INF("ModelRelease::ModelDestroy End ret = %d\n", bSuccess); - Print_MemInfo("ModelRelease::ModelDestroy End."); - - if (bSuccess) { - bSuccess = WriteFile(hSvcPipeOutWrite, ACTION_OK, (DWORD)strlen(ACTION_OK) + 1, NULL, NULL); - } - else { - bSuccess = WriteFile(hSvcPipeOutWrite, ACTION_FAILED, (DWORD)strlen(ACTION_FAILED) + 1, NULL, NULL); - } -} - -int svcprocess_run(HANDLE hSvcPipeInRead, HANDLE hSvcPipeOutWrite) { - DWORD dwRead = 0, dwWrite = 0; - BOOL bSuccess = false; - - if ((hSvcPipeOutWrite == INVALID_HANDLE_VALUE) || (hSvcPipeInRead == INVALID_HANDLE_VALUE)) { - ErrorExit("Svc::Failed to get write or read handle."); - } - - for (;;) { - bSuccess = ReadFile(hSvcPipeInRead, g_buffer, GLOBAL_BUFSIZE, &dwRead, NULL); - - if (!bSuccess || dwRead == 0) { - QNN_WAR("Svc::Failed to read from hSvcPipeInRead, perhaps parent process closed pipe or died.\n"); - break; - } - - char* cmdBuf = g_buffer + 1; - switch (g_buffer[0]) { - case 'l': // load model. - ModelLoad(cmdBuf, hSvcPipeOutWrite); - break; - - case 'g': // run Graphs. - ModelRun(cmdBuf, hSvcPipeOutWrite); - break; - - case 'r': // release model. - ModelRelease(cmdBuf, hSvcPipeOutWrite); - break; - } - } - - return 0; -} - - -// ============================== Client / SvcQNNHelper ============================== // -#define BUFSIZE (256) - -// test code, load and run model. -int hostprocess_run(std::string qnn_lib_path, std::string model_path, - std::string input_raw_path, int input_count, int memory_size, - std::string perf_profile) { - BOOL result = false; - - std::string MODEL_NAME = ""; - std::string PROC_NAME = ""; - - std::string model_memory_name = MODEL_NAME; - std::string model_name = MODEL_NAME; - std::string proc_name = PROC_NAME; - - std::string backend_lib_path = qnn_lib_path + "\\QnnHtp.dll"; - std::string system_lib_path = qnn_lib_path + "\\QnnSystem.dll"; - - std::string input_data_path = input_raw_path + "\\input_%d.raw"; - std::string output_data_path = input_raw_path + "\\output_%d.raw"; - - QNN_INF("Load data from raw data file to vector Start.\n"); - std::vector inputBuffers; - std::vector inputSize; - std::vector outputBuffers; - std::vector outputSize; - char dataPath[BUFSIZE]; - - for (int i = 0; i < input_count; i++) { - sprintf_s(dataPath, BUFSIZE, input_data_path.c_str(), i); - std::ifstream in(dataPath, std::ifstream::binary); - if (!in) { - QNN_ERR("Failed to open input file: %s", dataPath); - } - else { - uint8_t* buffer = nullptr; - in.seekg(0, in.end); - const size_t length = in.tellg(); - in.seekg(0, in.beg); - buffer = (uint8_t*)malloc(length); - if (!in.read(reinterpret_cast(buffer), length)) { - QNN_ERR("Failed to read the of: %s", dataPath); - } - - inputBuffers.push_back(reinterpret_cast(buffer)); - inputSize.push_back(length); - } - in.close(); - } - QNN_INF("Load data from raw data file to vector End.\n"); - - Print_MemInfo("Load data from raw data file End."); - - LibQNNHelper libQNNHelper; - - if (0 == memory_size) { // Load & run model locally. - QNN_INF("Load and run model locally Start.\n"); - result = libQNNHelper.ModelInitialize(model_name, model_path, backend_lib_path, system_lib_path); - Print_MemInfo("ModelInitialize End."); - - // SetPerfProfileGlobal("burst"); - - { - // Inference. - result = libQNNHelper.ModelInference(model_name, inputBuffers, outputBuffers, outputSize, perf_profile); - - // Verify the output data here. Free the data in vector. - for (int i = 0; i < outputSize.size(); i++) { - sprintf_s(dataPath, BUFSIZE, output_data_path.c_str(), i); - std::ofstream os(dataPath, std::ofstream::binary); - if (!os) { - QNN_ERR("Failed to open output file for writing: %s", dataPath); - } - else { - os.write(reinterpret_cast(&(*(outputBuffers[i]))), outputSize[i]); - } - os.close(); - } - - for (int i = 0; i < outputBuffers.size(); i++) { - free(outputBuffers[i]); - } - outputBuffers.clear(); - outputSize.clear(); - Print_MemInfo("ModelInference End."); - } - - result = libQNNHelper.ModelDestroy(model_name); - - QNN_INF("Load and run model locally End.\n"); - - Print_MemInfo("Load and run model locally End."); - } - else { // Load & run model in remote process. - libQNNHelper.CreateShareMemory(model_memory_name, memory_size); - - // Add 'TalkToSvc_*' function to 'libQNNHelper'. - QNN_INF("TalkToSvc_Initialize Start.\n"); - result = libQNNHelper.ModelInitialize(model_name, proc_name, model_path, backend_lib_path, system_lib_path); - QNN_INF("TalkToSvc_Initialize End %d.\n", result); - - QNN_INF("TalkToSvc_Inference Start.\n"); - result = libQNNHelper.ModelInference(model_name, proc_name, model_memory_name, inputBuffers, inputSize, outputBuffers, outputSize, perf_profile); - QNN_INF("TalkToSvc_Inference End %d.\n", result); - - // Verify the output data here. Free the data in vector. - for (int i = 0; i < outputSize.size(); i++) { - sprintf_s(dataPath, BUFSIZE, output_data_path.c_str(), i); - std::ofstream os(dataPath, std::ofstream::binary); - if (!os) { - QNN_ERR("Failed to open output file for writing: %s", dataPath); - } - else { - os.write(reinterpret_cast(&(*(outputBuffers[i]))), outputSize[i]); - } - os.close(); - } - - result = libQNNHelper.ModelDestroy(model_name, proc_name); - QNN_INF("TalkToSvc_Destroy End.\n"); - - libQNNHelper.DeleteShareMemory(model_memory_name); - QNN_INF("DeleteShareMem End.\n"); - - // outputBuffers is in ShareMemory, so we don't need to free this memory. - } - - // Release input buffer. - for (int i = 0; i < inputBuffers.size(); i++) { - free(inputBuffers[i]); - } - inputBuffers.clear(); - inputSize.clear(); - - return 0; -} - - -int main(int argc, char** argv) { - - if(argc > 1 && argv[1] && argv[1][0] == 's') { // Start server. - HANDLE hSvcPipeInRead = (HANDLE)std::stoull(argv[2]); - HANDLE hSvcPipeOutWrite = (HANDLE)std::stoull(argv[3]); - SetLogLevel(std::stoi(argv[5])); - SetProfilingLevel(std::stoi(argv[6])); - SetProcInfo(argv[7], std::stoull(argv[4])); - QNN_INF("Svc App Start proc %s.\n", argv[7]); - Print_MemInfo("Svc App Start."); - svcprocess_run(hSvcPipeInRead, hSvcPipeOutWrite); - Print_MemInfo("Svc App End."); - } - else { // Start test mode to load & run model. - // SvcQNNHelper.exe <1:int:log_level> <2:str:QNN_Libraries_Path> <3:str:model_path> <4:str:perf_profile> - // <5:str:input_raw_path> <6:int:input_count> <7:int:memory_size> - // input files are under 'input_raw_path' and the file names format are 'input_%d.raw'. - // SvcQNNHelper.exe 5 "" - - if (argc <= 6) { - printf("Command formant: SvcQNNHelper.exe <1:int:log_level> <2:str:QNN_Libraries_Path> <3:str:model_path> <4:str:perf_profile> <5:str:input_raw_path> <6:int:input_count> <7:int:memory_size>\n"); - printf("'memory_size' is an option parameter, only needed while running the model in remote process.\n"); - printf("Example: SvcQNNHelper.exe 5 \"C:\\test_model\\QNN_binaries\" \"C:\\test_model\\qnn_model\\InceptionV3_quantized.serialized.v73.bin\" \"bust\" \"C:\\test_model\\input\" 1 102400000"); - return 0; - } - - int log_level = std::stoi(argv[1]); - char* qnn_lib_path = argv[2]; - char* model_path = argv[3]; - char* perf_profile = argv[4]; - char* input_list_path = argv[5]; - int input_count = std::stoi(argv[6]); - int memory_size = 0; - if (argc > 7) { - memory_size = std::stoi(argv[7]); - } - - SetLogLevel(log_level); - - if(log_level >= 5) { - SetProfilingLevel(2); - } - else if(log_level >= 3) { - SetProfilingLevel(1); - } - - hostprocess_run(qnn_lib_path, model_path, input_list_path, input_count, memory_size, perf_profile); - - Print_MemInfo("Main App End."); - } - - return 0; -} -