From 54732036d830ce02ef1ced9877dc96daeeb7f429 Mon Sep 17 00:00:00 2001 From: MistEO Date: Wed, 8 Mar 2023 21:55:57 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E6=88=98=E6=96=97?= =?UTF-8?q?=E4=B8=AD=E6=8A=80=E8=83=BD=E8=AF=86=E5=88=AB=EF=BC=8C=E6=94=B9?= =?UTF-8?q?=E7=94=A8=E6=B7=B1=E5=BA=A6=E5=AD=A6=E4=B9=A0=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=20(#3918)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 战斗中技能使用ort识别 * fix: 修正技能识别的错误 * feat: 当技能识别置信度过低时，保存debug图片 * fix: build error for onnx runtime * fix: build error on linux * perf: onnx path * fix: build error * fix: onnx build error --- CMakeLists.txt | 22 +++- resource/tasks.json | 8 +- src/MaaCore/Config/OnnxSession.cpp | 20 +++ src/MaaCore/Config/OnnxSession.h | 24 ++++ src/MaaCore/Config/ResourceLoader.cpp | 4 +- src/MaaCore/MaaCore.vcxproj | 10 +- src/MaaCore/MaaCore.vcxproj.filters | 104 ++++++++------- .../Reclamation/ReclamationBattlePlugin.cpp | 96 +++++++------- .../BattleSkillReadyImageAnalyzer.cpp | 124 ++++++++++-------- .../BattleSkillReadyImageAnalyzer.h | 4 - 10 files changed, 248 insertions(+), 168 deletions(-) create mode 100644 src/MaaCore/Config/OnnxSession.cpp create mode 100644 src/MaaCore/Config/OnnxSession.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 654aa27163..9c5ecf326a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,7 +70,27 @@ find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs) find_package(ZLIB REQUIRED) find_package(MaaDerpLearning REQUIRED) find_package(asio REQUIRED) -target_link_libraries(MaaCore ${OpenCV_LIBS} MaaDerpLearning asio::asio ZLIB::ZLIB header_only_libraries) + +find_path(ONNXRuntime_INCLUDE_DIR NAMES onnxruntime/core/session/onnxruntime_c_api.h) +find_library(ONNXRuntime_LIBRARY NAMES onnxruntime) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + ONNXRuntime + REQUIRED_VARS ONNXRuntime_LIBRARY ONNXRuntime_INCLUDE_DIR +) +if(ONNXRuntime_FOUND) + set(ONNXRuntime_INCLUDE_DIRS ${ONNXRuntime_INCLUDE_DIR}) + if(NOT TARGET ONNXRuntime::ONNXRuntime) + add_library(ONNXRuntime::ONNXRuntime UNKNOWN IMPORTED) + set_target_properties(ONNXRuntime::ONNXRuntime PROPERTIES + IMPORTED_LOCATION "${ONNXRuntime_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${ONNXRuntime_INCLUDE_DIR}" + ) + endif() +endif() +find_package(ONNXRuntime) + +target_link_libraries(MaaCore ${OpenCV_LIBS} MaaDerpLearning asio::asio ZLIB::ZLIB ONNXRuntime::ONNXRuntime header_only_libraries) if(WIN32) target_link_libraries(MaaCore ws2_32) diff --git a/resource/tasks.json b/resource/tasks.json index aec92436cc..d51c1a1443 100644 --- a/resource/tasks.json +++ b/resource/tasks.json @@ -6264,10 +6264,10 @@ "BattleSkillReady": { "cache": false, "rectMove": [ - -20, - -125, - 45, - 50 + -28, + -140, + 64, + 64 ] }, "BattleSkillReadyOnClick": { diff --git a/src/MaaCore/Config/OnnxSession.cpp b/src/MaaCore/Config/OnnxSession.cpp new file mode 100644 index 0000000000..cbbdc721e6 --- /dev/null +++ b/src/MaaCore/Config/OnnxSession.cpp @@ -0,0 +1,20 @@ +#include "OnnxSession.h" + +#include +#include +#include + +#include "Utils/Logger.hpp" + +bool asst::OnnxSession::load(const std::filesystem::path& path) +{ + LogTraceFunction; + Log.info("load", path); + + std::string name = utils::path_to_utf8_string(path.stem()); + Ort::Session session(m_env, path.c_str(), m_options); + + m_sessions.insert_or_assign(std::move(name), std::move(session)); + + return true; +} diff --git a/src/MaaCore/Config/OnnxSession.h b/src/MaaCore/Config/OnnxSession.h new file mode 100644 index 0000000000..062f80fcdc --- /dev/null +++ b/src/MaaCore/Config/OnnxSession.h @@ -0,0 +1,24 @@ +#pragma once + +#include "AbstractResource.h" + +#include + +#include + +namespace asst +{ + class OnnxSession final : public SingletonHolder, public AbstractResource + { + public: + virtual ~OnnxSession() override = default; + virtual bool load(const std::filesystem::path& path) override; + + Ort::Session& get(const std::string& key) { return m_sessions.at(key); } + + private: + Ort::Env m_env; + Ort::SessionOptions m_options; + std::unordered_map m_sessions; + }; +} diff --git a/src/MaaCore/Config/ResourceLoader.cpp b/src/MaaCore/Config/ResourceLoader.cpp index 7203c27f65..803c9e7cc7 100644 --- a/src/MaaCore/Config/ResourceLoader.cpp +++ b/src/MaaCore/Config/ResourceLoader.cpp @@ -9,11 +9,12 @@ #include "Miscellaneous/CopilotConfig.h" #include "Miscellaneous/InfrastConfig.h" #include "Miscellaneous/ItemConfig.h" +#include "Miscellaneous/OcrConfig.h" #include "Miscellaneous/OcrPack.h" #include "Miscellaneous/RecruitConfig.h" #include "Miscellaneous/StageDropsConfig.h" #include "Miscellaneous/TilePack.h" -#include "Miscellaneous/OcrConfig.h" +#include "OnnxSession.h" #include "Roguelike/RoguelikeCopilotConfig.h" #include "Roguelike/RoguelikeRecruitConfig.h" #include "Roguelike/RoguelikeShoppingConfig.h" @@ -66,6 +67,7 @@ bool asst::ResourceLoader::load(const std::filesystem::path& path) LoadResourceAndCheckRet(RoguelikeShoppingConfig, "roguelike"_p / "shopping.json"_p); LoadResourceAndCheckRet(BattleDataConfig, "battle_data.json"_p); LoadResourceAndCheckRet(OcrConfig, "ocr_config.json"_p); + LoadResourceAndCheckRet(OnnxSession, "onnx"_p / "skill_ready_rec.onnx"_p); /* load resource with json and template files*/ LoadResourceWithTemplAndCheckRet(TaskData, "tasks.json"_p, "template"_p); diff --git a/src/MaaCore/MaaCore.vcxproj b/src/MaaCore/MaaCore.vcxproj index f5eeac53ee..3645a152e4 100644 --- a/src/MaaCore/MaaCore.vcxproj +++ b/src/MaaCore/MaaCore.vcxproj @@ -39,6 +39,7 @@ + @@ -173,6 +174,7 @@ + @@ -381,7 +383,7 @@ true true true - MaaDerpLearning.lib;opencv_world4.lib;zlib.lib;ws2_32.lib;%(AdditionalDependencies) + MaaDerpLearning.lib;opencv_world4.lib;zlib.lib;ws2_32.lib;onnxruntime.lib;%(AdditionalDependencies) RequireAdministrator true msvcrt.lib @@ -428,7 +430,7 @@ true true true - MaaDerpLearning.lib;opencv_world4.lib;zlib.lib;ws2_32.lib;%(AdditionalDependencies) + MaaDerpLearning.lib;opencv_world4.lib;zlib.lib;ws2_32.lib;onnxruntime.lib;%(AdditionalDependencies) RequireAdministrator true msvcrt.lib @@ -478,7 +480,7 @@ true true true - MaaDerpLearning.lib;opencv_world4.lib;zlib.lib;ws2_32.lib;%(AdditionalDependencies) + MaaDerpLearning.lib;opencv_world4.lib;zlib.lib;ws2_32.lib;onnxruntime.lib;%(AdditionalDependencies) RequireAdministrator true msvcrt.lib @@ -528,7 +530,7 @@ true true true - MaaDerpLearning.lib;opencv_world4.lib;zlib.lib;ws2_32.lib;%(AdditionalDependencies) + MaaDerpLearning.lib;opencv_world4.lib;zlib.lib;ws2_32.lib;onnxruntime.lib;%(AdditionalDependencies) RequireAdministrator true msvcrt.lib diff --git a/src/MaaCore/MaaCore.vcxproj.filters b/src/MaaCore/MaaCore.vcxproj.filters index a3ff708973..af39ff2263 100644 --- a/src/MaaCore/MaaCore.vcxproj.filters +++ b/src/MaaCore/MaaCore.vcxproj.filters @@ -26,15 +26,6 @@ {250bc7cb-e99a-42f8-ae3c-66998564fdcf} - - {3b2a04b8-fb22-43c3-9444-1b455851d8e2} - - - {de1acbf1-3e7a-4e91-8a62-e966602c6b25} - - - {64367a8b-0654-47bb-92cc-d451799457a4} - {8499a77f-7bcc-457c-97f1-6e950ab68bf4} @@ -86,6 +77,15 @@ {a61a19a3-e1ba-45f2-9264-acb4102106b9} + + {3b2a04b8-fb22-43c3-9444-1b455851d8e2} + + + {de1acbf1-3e7a-4e91-8a62-e966602c6b25} + + + {64367a8b-0654-47bb-92cc-d451799457a4} + @@ -196,52 +196,52 @@ 源文件\Vision\Roguelike - 源文件\Config + 源文件\Resource - 源文件\Config + 源文件\Resource - 源文件\Config + 源文件\Resource - 源文件\Config + 源文件\Resource - 源文件\Config + 源文件\Resource - 源文件\Config + 源文件\Resource - 源文件\Config\Roguelike + 源文件\Resource\Roguelike - 源文件\Config\Roguelike + 源文件\Resource\Roguelike - 源文件\Config\Roguelike + 源文件\Resource\Roguelike - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous 源文件\Task @@ -355,10 +355,10 @@ 源文件 - 源文件\Config + 源文件\Resource - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous 源文件\Task\Fight @@ -469,13 +469,13 @@ 源文件\Task - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous 源文件\Task\Interface - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous 源文件\Task\SSS @@ -523,7 +523,10 @@ 源文件\Controller\adb-lite - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous + + + 源文件\Resource @@ -591,46 +594,46 @@ 源文件\Vision\Roguelike - 源文件\Config + 源文件\Resource - 源文件\Config + 源文件\Resource - 源文件\Config + 源文件\Resource - 源文件\Config + 源文件\Resource - 源文件\Config\Roguelike + 源文件\Resource\Roguelike - 源文件\Config\Roguelike + 源文件\Resource\Roguelike - 源文件\Config\Roguelike + 源文件\Resource\Roguelike - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous 源文件\Task @@ -696,10 +699,10 @@ 源文件 - 源文件\Config + 源文件\Resource - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous 源文件\Task\Fight @@ -813,13 +816,13 @@ 源文件\Task - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous 源文件\Task\Interface - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous 源文件\Task\SSS @@ -861,7 +864,10 @@ 源文件\Controller\adb-lite - 源文件\Config\Miscellaneous + 源文件\Resource\Miscellaneous + + + 源文件\Resource \ No newline at end of file diff --git a/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp b/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp index bef93b21df..ff665f98b5 100644 --- a/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp +++ b/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp @@ -2,8 +2,8 @@ #include "Utils/NoWarningCV.h" -#include "Controller/Controller.h" #include "Config/TaskData.h" +#include "Controller/Controller.h" #include "ReclamationControlTask.h" #include "Status.h" #include "Task/ProcessTask.h" @@ -123,52 +123,54 @@ bool asst::ReclamationBattlePlugin::communicate_with(const std::string& npcName) bool asst::ReclamationBattlePlugin::communicate_with_aux( const std::string& npcName, std::function orderComp) { - auto image = ctrler()->get_image(); - BattleSkillReadyImageAnalyzer skillReadyAnalyzer(image); - if (!skillReadyAnalyzer.analyze()) { - Log.info(__FUNCTION__, " | ", "no ready skills"); - return false; - } - std::vector skill_results = skillReadyAnalyzer.get_result(); - - std::sort(skill_results.begin(), skill_results.end(), orderComp); - for (const auto& [score, rect] : skill_results) { - Rect center(rect.x + rect.width / 2, - rect.y + rect.height / 2 + Task.get("Reclamation@SkillReadyRoleOffset")->special_params.front(), 5, - 5); - - const auto use_oper_task_ptr = Task.get("BattleUseOper"); - ctrler()->click(center); - sleep(use_oper_task_ptr->pre_delay); - - image = ctrler()->get_image(); - OcrImageAnalyzer npcNameAnalyzer(image); - npcNameAnalyzer.set_task_info("Reclamation@Liaison"); - npcNameAnalyzer.set_required({}); - if (!npcNameAnalyzer.analyze()) { - // 地图发生了移动 - Log.info(__FUNCTION__, " | ", "map moved "); - break; - } - if (npcNameAnalyzer.get_result().front() != npcName) { - // npc名称不正确 - Log.info(__FUNCTION__, " | ", "npc name not match ", npcNameAnalyzer.get_result().front()); - cancel_oper_selection(); - continue; - } - - ProcessTask skill_task(this_task(), { "Reclamation@BattleSkillReadyOnClick" }); - skill_task.set_task_delay(0); - - bool ret = skill_task.set_retry_times(5).run(); - if (!ret) { - cancel_oper_selection(); - Log.info(__FUNCTION__, " | ", "fail to click skill of npc"); - return false; - } - - return true; - } + std::ignore = npcName; + std::ignore = orderComp; + // auto image = ctrler()->get_image(); + // BattleSkillReadyImageAnalyzer skillReadyAnalyzer(image); + // if (!skillReadyAnalyzer.analyze()) { + // Log.info(__FUNCTION__, " | ", "no ready skills"); + // return false; + // } + // std::vector skill_results = skillReadyAnalyzer.get_result(); + // + // std::sort(skill_results.begin(), skill_results.end(), orderComp); + // for (const auto& [score, rect] : skill_results) { + // Rect center(rect.x + rect.width / 2, + // rect.y + rect.height / 2 + + // Task.get("Reclamation@SkillReadyRoleOffset")->special_params.front(), 5, 5); + // + // const auto use_oper_task_ptr = Task.get("BattleUseOper"); + // ctrler()->click(center); + // sleep(use_oper_task_ptr->pre_delay); + // + // image = ctrler()->get_image(); + // OcrImageAnalyzer npcNameAnalyzer(image); + // npcNameAnalyzer.set_task_info("Reclamation@Liaison"); + // npcNameAnalyzer.set_required({}); + // if (!npcNameAnalyzer.analyze()) { + // // 地图发生了移动 + // Log.info(__FUNCTION__, " | ", "map moved "); + // break; + // } + // if (npcNameAnalyzer.get_result().front() != npcName) { + // // npc名称不正确 + // Log.info(__FUNCTION__, " | ", "npc name not match ", npcNameAnalyzer.get_result().front()); + // cancel_oper_selection(); + // continue; + // } + // + // ProcessTask skill_task(this_task(), { "Reclamation@BattleSkillReadyOnClick" }); + // skill_task.set_task_delay(0); + // + // bool ret = skill_task.set_retry_times(5).run(); + // if (!ret) { + // cancel_oper_selection(); + // Log.info(__FUNCTION__, " | ", "fail to click skill of npc"); + // return false; + // } + // + // return true; + // } return false; } diff --git a/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.cpp b/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.cpp index 510d947830..5a92869b8b 100644 --- a/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.cpp @@ -3,76 +3,84 @@ #include "Utils/NoWarningCV.h" #include -#include -#include +#include +#include +#include + +#include "Config/OnnxSession.h" #include "Config/TaskData.h" -#include "Config/TemplResource.h" #include "Utils/Logger.hpp" +template +static void softmax(T& input) +{ + float rowmax = *std::max_element(input.begin(), input.end()); + std::vector y(input.size()); + float sum = 0.0f; + for (size_t i = 0; i != input.size(); ++i) { + sum += y[i] = std::exp(input[i] - rowmax); + } + for (size_t i = 0; i != input.size(); ++i) { + input[i] = y[i] / sum; + } +} + bool asst::BattleSkillReadyImageAnalyzer::analyze() { - auto task_ptr = Task.get("BattleSkillReady"); - const cv::Mat& templ = TemplResource::get_instance().get_templ(task_ptr->templ_name); + LogTraceFunction; - auto key_color = [](cv::InputArray src, cv::OutputArray dst, const cv::Scalar& color, - const cv::Scalar& tolerance = {}) { - cv::inRange(src, color - tolerance, color + tolerance, dst); - }; - auto preprocess = [&](cv::Mat& img) { - cv::Mat mask_y2; - key_color(img, mask_y2, { 40, 94, 103 }, { 10, 10, 10 }); // BGR - img.setTo(cv::Scalar { 2, 216, 255 }, mask_y2); // BGR - cv::Mat mask_y1; - key_color(img, mask_y1, { 2, 216, 255 }, { 20, 20, 20 }); - cv::dilate(mask_y1, mask_y1, cv::getStructuringElement(cv::MORPH_RECT, { 4, 4 })); - cv::bitwise_not(mask_y1, mask_y1); - img.setTo(cv::Scalar { 0, 0, 0 }, mask_y1); - }; + cv::Mat image = m_image(make_rect(m_roi)); + cv::cvtColor(image, image, cv::COLOR_BGR2RGB); - cv::Mat tmp = templ.clone(); - cv::Mat img = m_image(make_rect(m_roi)).clone(); - - cv::Mat template_mask; - key_color(tmp, template_mask, { 0, 255, 0 }, { 0, 0, 0 }); - cv::bitwise_not(template_mask, template_mask); - - preprocess(img); - preprocess(tmp); - - cv::Mat match; - cv::matchTemplate(img, tmp, match, cv::TM_SQDIFF, template_mask); - match /= template_mask.cols * template_mask.rows; - cv::sqrt(match, match); - - // 修改自MultiMatchImageAnalyzer - const double templ_thres = 130.; - int mini_distance = (std::min)(templ.cols, templ.rows) / 2; - for (int i = 0; i != match.rows; ++i) { - for (int j = 0; j != match.cols; ++j) { - auto value = match.at(i, j); - if (value < templ_thres) { - Rect rect(j + m_roi.x, i + m_roi.y, templ.cols, templ.rows); - bool need_push = true; - for (auto& iter : ranges::reverse_view(m_result)) { - if (std::abs(j + m_roi.x - iter.rect.x) < mini_distance && - std::abs(i + m_roi.y - iter.rect.y) < mini_distance) { - if (iter.score > value) { - iter.rect = rect; - iter.score = value; - } - need_push = false; - break; - } - } - if (need_push) { - m_result.emplace_back(value, rect); - } + size_t input_size = 1ULL * image.cols * image.rows * image.channels(); + std::vector input(input_size); + std::vector output; + size_t count = 0; + for (int k = 0; k < image.channels(); k++) { + for (int i = 0; i < image.rows; i++) { + for (int j = 0; j < image.cols; j++) { + float value = static_cast(image.at(i, j)[k]); + value /= 255; + value -= 0.5f; // mean + value /= 0.5f; // std + input[count++] = value; } } } - return !m_result.empty(); + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + constexpr int64_t batch_size = 1; + std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; + + Ort::Value input_tensor = + Ort::Value::CreateTensor(memory_info, input.data(), input_size, input_shape.data(), input_shape.size()); + + constexpr size_t classification_size = 2; + std::array results; + std::array output_shape { batch_size, classification_size }; + Ort::Value output_tensor = Ort::Value::CreateTensor(memory_info, results.data(), results.size(), + output_shape.data(), output_shape.size()); + + auto& session = OnnxSession::get_instance().get("skill_ready_rec"); + // 这俩是hardcode在模型里的 + constexpr const char* input_names[] = { "input" }; // session.GetInputName() + constexpr const char* output_names[] = { "output" }; // session.GetOutputName() + + Ort::RunOptions run_options; + session.Run(run_options, input_names, &input_tensor, 1, output_names, &output_tensor, 1); + + softmax(results); + + Log.info("after softmax, 0: ", results[0], ", 1: ", results[1]); + + auto max_iter = std::max_element(results.begin(), results.end()); + if (*max_iter < 0.7) { + Log.warn("Skill ready recognition confidence too low: ", *max_iter, ", roi:", m_roi); + save_img(utils::path("debug") / utils::path("skill_ready_rec")); + } + + return results[1] > results[0]; } void asst::BattleSkillReadyImageAnalyzer::set_base_point(const Point& pt) diff --git a/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h b/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h index 8b5927aa11..1ff0894a43 100644 --- a/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h +++ b/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h @@ -11,11 +11,7 @@ namespace asst virtual ~BattleSkillReadyImageAnalyzer() override = default; virtual bool analyze() override; - const std::vector& get_result() const noexcept { return m_result; } void set_base_point(const Point& pt); - - private: - std::vector m_result; }; }