diff --git a/src/MaaCore/MaaCore.vcxproj b/src/MaaCore/MaaCore.vcxproj index e2ea60b4ec..1c19fc39e1 100644 --- a/src/MaaCore/MaaCore.vcxproj +++ b/src/MaaCore/MaaCore.vcxproj @@ -83,11 +83,10 @@ - - + - - + + @@ -236,11 +235,10 @@ - - + - - + + diff --git a/src/MaaCore/MaaCore.vcxproj.filters b/src/MaaCore/MaaCore.vcxproj.filters index f95d5fe6ad..d1bcc174e7 100644 --- a/src/MaaCore/MaaCore.vcxproj.filters +++ b/src/MaaCore/MaaCore.vcxproj.filters @@ -567,19 +567,16 @@ 源文件\Task\Interface - + 源文件\Vision\Battle 源文件\Vision\Battle - + 源文件\Vision\Battle - - 源文件\Vision\Battle - - + 源文件\Vision\Battle @@ -968,19 +965,16 @@ 源文件\Task\Interface - + 源文件\Vision\Battle 源文件\Vision\Battle - + 源文件\Vision\Battle - - 源文件\Vision\Battle - - + 源文件\Vision\Battle diff --git a/src/MaaCore/Task/BattleHelper.cpp b/src/MaaCore/Task/BattleHelper.cpp index 2bb2932c1d..535204b955 100644 --- a/src/MaaCore/Task/BattleHelper.cpp +++ b/src/MaaCore/Task/BattleHelper.cpp @@ -11,7 +11,7 @@ #include "Utils/ImageIo.hpp" #include "Utils/Logger.hpp" #include "Utils/NoWarningCV.h" -#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" +#include "Vision/Battle/BattlefieldClassifier.h" #include "Vision/Battle/BattlefieldMatcher.h" #include "Vision/BestMatcher.h" #include "Vision/Matcher.h" @@ -476,7 +476,8 @@ bool asst::BattleHelper::check_and_use_skill(const std::string& name, bool& has_ bool asst::BattleHelper::check_and_use_skill(const Point& loc, bool& has_error, const cv::Mat& reusable) { cv::Mat image = reusable.empty() ? m_inst_helper.ctrler()->get_image() : reusable; - BattleSkillReadyImageAnalyzer skill_analyzer(image); + BattlefieldClassifier skill_analyzer(image); + skill_analyzer.set_object_of_interest({ .skill_ready = true }); auto target_iter = m_normal_tile_info.find(loc); if (target_iter == m_normal_tile_info.end()) { @@ -485,7 +486,7 @@ bool asst::BattleHelper::check_and_use_skill(const Point& loc, bool& has_error, } const Point& battlefield_point = target_iter->second.pos; skill_analyzer.set_base_point(battlefield_point); - if (!skill_analyzer.analyze()) { + if (!skill_analyzer.analyze()->skill_ready.ready) { return false; } diff --git a/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp b/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp index e41f8416e1..3a46b381c6 100644 --- a/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp +++ b/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp @@ -7,10 +7,9 @@ #include "Utils/Logger.hpp" #include "Utils/NoWarningCV.h" #include "Utils/Ranges.hpp" -#include "Vision/Battle/BattleDeployDirectionImageAnalyzer.h" -#include "Vision/Battle/BattleFormationImageAnalyzer.h" -#include "Vision/Battle/BattleOperatorsImageAnalyzer.h" -#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" +#include "Vision/Battle/BattleFormationAnalyzer.h" +#include "Vision/Battle/BattlefieldClassifier.h" +#include "Vision/Battle/BattlefieldDetector.h" #include "Vision/Battle/BattlefieldMatcher.h" #include "Vision/BestMatcher.h" #include "Vision/RegionOCRer.h" @@ -117,7 +116,7 @@ bool asst::CombatRecordRecognitionTask::analyze_formation() const int skip_count = m_video_fps > m_formation_fps ? static_cast(m_video_fps / m_formation_fps) - 1 : 0; - BattleFormationImageAnalyzer formation_ananlyzer; + BattleFormationAnalyzer formation_ananlyzer; int no_changes_count = 0; for (size_t i = 0; i < m_video_frame_count; i += skip_frames(skip_count) + 1) { cv::Mat frame; @@ -131,13 +130,14 @@ bool asst::CombatRecordRecognitionTask::analyze_formation() cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); formation_ananlyzer.set_image(frame); - bool analyzed = formation_ananlyzer.analyze(); + auto formation_opt = formation_ananlyzer.analyze(); show_img(formation_ananlyzer); // 有些视频会有个过渡或者动画啥的，只取一帧识别的可能不全。多识别几帧 - if (analyzed) { - const auto& cur = formation_ananlyzer.get_result(); - if (cur.size() > m_formation.size()) { - m_formation = cur; + if (formation_opt) { + if (formation_opt->size() > m_formation.size()) { + for (const auto& [name, avatar] : *formation_opt) { + m_formation.insert_or_assign(name, avatar); + } } else if (++no_changes_count > 5) { m_formation_end_frame = i; @@ -509,9 +509,10 @@ bool asst::CombatRecordRecognitionTask::compare_skill(ClipInfo& clip, ClipInfo& const std::string oper_name = pre_clip.ends_oper_name; const Point target_location = m_operator_locations[oper_name]; const Point target_position = m_normal_tile_info[target_location].pos; - BattleSkillReadyImageAnalyzer analyzer(pre_clip.end_frame); + BattlefieldClassifier analyzer(pre_clip.end_frame); + analyzer.set_object_of_interest({ .skill_ready = true }); analyzer.set_base_point(target_position); - bool pre_ready = analyzer.analyze(); + bool pre_ready = analyzer.analyze()->skill_ready.ready; show_img(analyzer); if (!pre_ready) { @@ -539,7 +540,7 @@ bool asst::CombatRecordRecognitionTask::compare_skill(ClipInfo& clip, ClipInfo& } cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); analyzer.set_image(frame); - bool cur_ready = analyzer.analyze(); + bool cur_ready = analyzer.analyze()->skill_ready.ready; if (pre_ready && !cur_ready) { json::object condition = analyze_action_condition(clip, &pre_clip); @@ -590,13 +591,14 @@ bool asst::CombatRecordRecognitionTask::detect_operators(ClipInfo& clip, [[maybe } cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); - BattleOperatorsImageAnalyzer analyzer(frame); - analyzer.analyze(); + BattlefieldDetector analyzer(frame); + analyzer.set_object_of_interest({ .operators = true }); + auto result_opt = analyzer.analyze(); show_img(analyzer); DetectionResult cur_locations; auto tiles = m_normal_tile_info | views::values; - for (const auto& box : analyzer.get_results()) { + for (const auto& box : result_opt->operators) { Rect rect = box.rect.move(det_box_move); auto iter = ranges::find_if(tiles, [&](const TilePack::TileInfo& t) { return rect.include(t.pos); }); if (iter == tiles.end()) { @@ -650,18 +652,19 @@ bool asst::CombatRecordRecognitionTask::classify_direction(ClipInfo& clip, ClipI callback(AsstMsg::SubTaskStart, basic_info_with_what("ClassifyDirection")); /* classify direction */ - using Losses = BattleDeployDirectionImageAnalyzer::RawResults; - constexpr size_t ClsSize = BattleDeployDirectionImageAnalyzer::ClassificationSize; - std::unordered_map dir_cls_sampling; + using Raw = BattlefieldClassifier::DeployDirectionResult::Raw; + constexpr size_t ClsSize = BattlefieldClassifier::DeployDirectionResult::ClsSize; + std::unordered_map dir_cls_sampling; for (const cv::Mat& frame : clip.random_frames) { - BattleDeployDirectionImageAnalyzer analyzer(frame); + BattlefieldClassifier analyzer(frame); + analyzer.set_object_of_interest({ .skill_ready = false, .deploy_direction = true }); for (const auto& loc : newcomer) { analyzer.set_base_point(m_normal_tile_info.at(loc).pos); - analyzer.analyze(); + auto result_opt = analyzer.analyze(); show_img(analyzer); for (size_t i = 0; i < ClsSize; ++i) { - dir_cls_sampling[loc][i] += analyzer.get_raw_results()[i]; + dir_cls_sampling[loc][i] += result_opt->deploy_direction.raw[i]; } } } diff --git a/src/MaaCore/Task/Interface/DebugTask.cpp b/src/MaaCore/Task/Interface/DebugTask.cpp index 46990e6a1c..241423bcd6 100644 --- a/src/MaaCore/Task/Interface/DebugTask.cpp +++ b/src/MaaCore/Task/Interface/DebugTask.cpp @@ -6,7 +6,7 @@ #include "Utils/ImageIo.hpp" #include "Utils/Logger.hpp" -#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" +#include "Vision/Battle/BattlefieldClassifier.h" #include "Vision/Battle/BattlefieldMatcher.h" #include "Vision/Miscellaneous/DepotImageAnalyzer.h" #include "Vision/Miscellaneous/StageDropsImageAnalyzer.h" @@ -42,17 +42,19 @@ void asst::DebugTask::test_skill_ready() int correct = 0; for (const auto& entry : std::filesystem::directory_iterator(R"(../../test/skill_ready/y)")) { cv::Mat image = imread(entry.path().string()); - BattleSkillReadyImageAnalyzer analyzer(image); + BattlefieldClassifier analyzer(image); + analyzer.set_object_of_interest({ .skill_ready = true }); total++; - if (analyzer.analyze()) { + if (analyzer.analyze()->skill_ready.ready) { correct++; } } for (const auto& entry : std::filesystem::directory_iterator(R"(../../test/skill_ready/n)")) { cv::Mat image = imread(entry.path().string()); - BattleSkillReadyImageAnalyzer analyzer(image); + BattlefieldClassifier analyzer(image); + analyzer.set_object_of_interest({ .skill_ready = true }); total++; - if (!analyzer.analyze()) { + if (!analyzer.analyze()->skill_ready.ready) { correct++; } } diff --git a/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp b/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp index b38b237541..2af92a9ca3 100644 --- a/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp +++ b/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp @@ -8,7 +8,6 @@ #include "Status.h" #include "Task/ProcessTask.h" #include "Utils/Logger.hpp" -#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" #include "Vision/Matcher.h" #include "Vision/OCRer.h" @@ -126,7 +125,8 @@ bool asst::ReclamationBattlePlugin::communicate_with_aux( std::ignore = npcName; std::ignore = orderComp; // auto image = ctrler()->get_image(); - // BattleSkillReadyImageAnalyzer skillReadyAnalyzer(image); + // BattlefieldClassifier skillReadyAnalyzer(image); + // analyzer.set_object_of_interest({ .skill_ready = true }); // if (!skillReadyAnalyzer.analyze()) { // Log.info(__FUNCTION__, " | ", "no ready skills"); // return false; diff --git a/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.cpp deleted file mode 100644 index 41718abfd4..0000000000 --- a/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include "BattleDeployDirectionImageAnalyzer.h" - -#include "Utils/NoWarningCV.h" - -#include -#include -#include - -#include - -#include "Config/OnnxSessions.h" -#include "Config/TaskData.h" -#include "Utils/Logger.hpp" - -bool asst::BattleDeployDirectionImageAnalyzer::analyze() -{ - cv::Mat image = m_image(make_rect(m_roi)); - std::vector input = image_to_tensor(image); - - auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); - constexpr int64_t batch_size = 1; - std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; - - Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, input.data(), input.size(), - input_shape.data(), input_shape.size()); - - RawResults results; - std::array output_shape { batch_size, ClassificationSize }; - Ort::Value output_tensor = Ort::Value::CreateTensor(memory_info, results.data(), results.size(), - output_shape.data(), output_shape.size()); - - auto& session = OnnxSessions::get_instance().get("deploy_direction_cls"); - // 这俩是hardcode在模型里的 - constexpr const char* input_names[] = { "input" }; // session.GetInputName() - constexpr const char* output_names[] = { "output" }; // session.GetOutputName() - - Ort::RunOptions run_options; - session.Run(run_options, input_names, &input_tensor, 1, output_names, &output_tensor, 1); - m_raw_results = results; - Log.info(__FUNCTION__, "raw result:", m_raw_results); - - softmax(results); - Log.info(__FUNCTION__, "after softmax:", results); - - m_class_id = std::max_element(results.begin(), results.end()) - results.begin(); - -#ifdef ASST_DEBUG - static const std::unordered_map ClassNames = { - { 0, "Right" }, - { 1, "Down" }, - { 2, "Left" }, - { 3, "Up" }, - }; - cv::putText(m_image_draw, ClassNames.at(m_class_id), cv::Point(m_roi.x, m_roi.y + m_roi.height), - cv::FONT_HERSHEY_PLAIN, 1.2, cv::Scalar(0, 255, 0), 2); - cv::putText(m_image_draw, std::to_string(results[m_class_id]), cv::Point(m_roi.x, m_roi.y + m_roi.height + 20), - cv::FONT_HERSHEY_PLAIN, 1.2, cv::Scalar(0, 255, 0), 2); -#endif - - return true; -} - -void asst::BattleDeployDirectionImageAnalyzer::set_base_point(const Point& pt) -{ - auto task_ptr = Task.get("BattleDeployDirectionRectMove"); - const Rect& roi_move = task_ptr->rect_move; - - set_roi(Rect(pt.x, pt.y, 0, 0).move(roi_move)); -} diff --git a/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.h deleted file mode 100644 index a3fe1563cf..0000000000 --- a/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include "Vision/OnnxHelper.h" - -namespace asst -{ - class BattleDeployDirectionImageAnalyzer final : public OnnxHelper - { - public: - static constexpr size_t ClassificationSize = 4; - using RawResults = std::array; - - public: - using OnnxHelper::OnnxHelper; - virtual ~BattleDeployDirectionImageAnalyzer() override = default; - - void set_base_point(const Point& pt); - bool analyze(); - - size_t get_class_id() const { return m_class_id; } - RawResults get_raw_results() const { return m_raw_results; } - - private: - std::array m_raw_results; - size_t m_class_id = 0; - }; -} diff --git a/src/MaaCore/Vision/Battle/BattleFormationAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleFormationAnalyzer.cpp new file mode 100644 index 0000000000..08090c47ad --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattleFormationAnalyzer.cpp @@ -0,0 +1,76 @@ +#include "BattleFormationAnalyzer.h" + +#include "Config/Miscellaneous/BattleDataConfig.h" +#include "Config/TaskData.h" +#include "Utils/Logger.hpp" +#include "Utils/NoWarningCV.h" +#include "Vision/Miscellaneous/PipelineAnalyzer.h" +#include "Vision/TemplDetOCRer.h" + +using namespace asst; + +BattleFormationAnalyzer::ResultsVecOpt BattleFormationAnalyzer::analyze() const +{ + LogTraceFunction; + + PipelineAnalyzer start_button_analyzer(m_image); + start_button_analyzer.set_tasks(Task.get("BattleStartAll")->next); + if (!start_button_analyzer.analyze()) { + return std::nullopt; + } + + TemplDetOCRer::ResultsVec oper_names; + + TemplDetOCRer oper_names_analyzer(m_image); + oper_names_analyzer.set_task_info("BattleFormationOCRNameFlag", "BattleFormationOperNames"); + oper_names_analyzer.set_bin_expansion(3); + auto oper_names_opt = oper_names_analyzer.analyze(); + if (!oper_names_opt) { + return std::nullopt; + } + ResultsVec results = proc_ocr_result(*oper_names_opt); + + // 识别到了结果，但是确实没有可用的干员名。说明游戏UI可能是老版本的，布局不一样 + if (results.empty()) { + oper_names_analyzer.set_task_info("BattleFormationOCRNameFlag", "BattleFormationOperNamesOldVersion"); + oper_names_analyzer.set_bin_expansion(2); + auto old_layout_oper_names_opt = oper_names_analyzer.analyze(); + if (!old_layout_oper_names_opt) { + return std::nullopt; + } + results = proc_ocr_result(*old_layout_oper_names_opt); + } + + if (results.empty()) { + return std::nullopt; + } + + return results; +} + +BattleFormationAnalyzer::ResultsVec BattleFormationAnalyzer::proc_ocr_result( + const TemplDetOCRer::ResultsVec& ocr_result) const +{ + ResultsVec results; + + for (const auto& name_res : ocr_result) { + if (BattleData.is_name_invalid(name_res.text)) { + continue; + } + + const std::string& name = name_res.text; + Rect base_point { name_res.rect.x + name_res.rect.width, name_res.rect.y, 0, 0 }; + Rect avatar_rect = base_point.move(Task.get("BattleFormationOperAvatarMove")->rect_move); + cv::Mat avatar = m_image(make_rect(avatar_rect)); + results.emplace_back(Result { .name = name, .avatar = avatar }); + +#ifdef ASST_DEBUG + cv::rectangle(m_image_draw, make_rect(name_res.rect), cv::Scalar(0, 255, 0), 2); + cv::rectangle(m_image_draw, make_rect(avatar_rect), cv::Scalar(0, 0, 255), 2); + cv::putText(m_image_draw, BattleData.get_id(name), cv::Point(avatar_rect.x, avatar_rect.y - 20), 1, 1.2, + cv::Scalar(0, 0, 255), 2); +#endif + } + + return results; +} diff --git a/src/MaaCore/Vision/Battle/BattleFormationAnalyzer.h b/src/MaaCore/Vision/Battle/BattleFormationAnalyzer.h new file mode 100644 index 0000000000..21a72fb267 --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattleFormationAnalyzer.h @@ -0,0 +1,31 @@ +#pragma once + +#include "Vision/TemplDetOCRer.h" +#include "Vision/VisionHelper.h" + +namespace asst +{ + class BattleFormationAnalyzer : public VisionHelper + { + public: + struct Result + { + std::string to_string() const { return name; } + explicit operator std::string() const { return to_string(); } + + std::string name; + cv::Mat avatar; + }; + using ResultsVec = std::vector; + using ResultsVecOpt = std::optional; + + public: + using VisionHelper::VisionHelper; + virtual ~BattleFormationAnalyzer() override = default; + + ResultsVecOpt analyze() const; + + protected: + ResultsVec proc_ocr_result(const TemplDetOCRer::ResultsVec& ocr_result) const; + }; +} diff --git a/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.cpp deleted file mode 100644 index d01f8b8a74..0000000000 --- a/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "BattleFormationImageAnalyzer.h" - -#include "Config/Miscellaneous/BattleDataConfig.h" -#include "Config/TaskData.h" -#include "Utils/Logger.hpp" -#include "Utils/NoWarningCV.h" -#include "Vision/Miscellaneous/PipelineAnalyzer.h" -#include "Vision/TemplDetOCRer.h" - -bool asst::BattleFormationImageAnalyzer::analyze() -{ - LogTraceFunction; - m_result.clear(); - - PipelineAnalyzer start_button_analyzer(m_image); - start_button_analyzer.set_tasks(Task.get("BattleStartAll")->next); - if (!start_button_analyzer.analyze()) { - return false; - } - - TemplDetOCRer oper_names_analyzer(m_image); - oper_names_analyzer.set_task_info("BattleFormationOCRNameFlag", "BattleFormationOperNames"); - oper_names_analyzer.set_bin_expansion(3); - if (!oper_names_analyzer.analyze()) { - return false; - } - - auto proc_ocr_result = [&]() { - const auto& oper_names = oper_names_analyzer.get_result(); - for (const auto& name_res : oper_names) { - if (BattleData.is_name_invalid(name_res.text)) { - continue; - } - const std::string& name = name_res.text; - Rect base_point { name_res.rect.x + name_res.rect.width, name_res.rect.y, 0, 0 }; - Rect avatar_rect = base_point.move(Task.get("BattleFormationOperAvatarMove")->rect_move); - cv::Mat avatar = m_image(make_rect(avatar_rect)); - m_result.emplace(name, avatar); -#ifdef ASST_DEBUG - cv::rectangle(m_image_draw, make_rect(name_res.rect), cv::Scalar(0, 255, 0), 2); - cv::rectangle(m_image_draw, make_rect(avatar_rect), cv::Scalar(0, 0, 255), 2); - cv::putText(m_image_draw, utils::utf8_to_ansi(name), cv::Point(avatar_rect.x, avatar_rect.y - 20), 1, 1.2, - cv::Scalar(0, 0, 255), 2); -#endif - } - }; - - proc_ocr_result(); - - // 识别到了结果，但是确实没有可用的干员名。说明游戏UI可能是老版本的，布局不一样 - if (m_result.empty()) { - oper_names_analyzer.set_task_info("BattleFormationOCRNameFlag", "BattleFormationOperNamesOldVersion"); - oper_names_analyzer.set_bin_expansion(2); - if (!oper_names_analyzer.analyze()) { - return false; - } - proc_ocr_result(); - } - - return !m_result.empty(); -} diff --git a/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.h deleted file mode 100644 index 6020785dda..0000000000 --- a/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "Vision/VisionHelper.h" - -namespace asst -{ - class BattleFormationImageAnalyzer : public VisionHelper - { - public: - using VisionHelper::VisionHelper; - virtual ~BattleFormationImageAnalyzer() override = default; - - bool analyze(); - - const auto& get_result() const { return m_result; } - - private: - std::unordered_map m_result; - }; -} diff --git a/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.cpp index b24642460d..9a3a310d03 100644 --- a/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.cpp @@ -16,10 +16,9 @@ bool asst::BattleOperatorsImageAnalyzer::analyze() { LogTraceFunction; - m_results.clear(); - double x_scale = 640.0 / m_image.cols; - double y_scale = 640.0 / m_image.rows; + const double x_scale = 640.0 / m_image.cols; + const double y_scale = 640.0 / m_image.rows; cv::Mat image; cv::resize(m_image, image, cv::Size(), x_scale, y_scale, cv::INTER_AREA); diff --git a/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.h deleted file mode 100644 index bb7f25860e..0000000000 --- a/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include "Vision/OnnxHelper.h" - -namespace asst -{ - // 战斗中干员（血条）检测，yolov8 模型 - class BattleOperatorsImageAnalyzer final : public OnnxHelper - { - public: - enum class Cls - { - Operator = 0, - }; - struct Box - { - Cls cls = Cls::Operator; - Rect rect; - float score = .0f; - }; - - public: - using OnnxHelper::OnnxHelper; - virtual ~BattleOperatorsImageAnalyzer() override = default; - - bool analyze(); - - const std::vector& get_results() const { return m_results; } -#ifdef ASST_DEBUG - const std::vector& get_draw_rect() const { return m_draw_rect; } -#endif - - private: - std::vector m_results; - -#ifdef ASST_DEBUG - std::vector m_draw_rect; -#endif - }; -} diff --git a/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.cpp deleted file mode 100644 index 1a8466a587..0000000000 --- a/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include "BattleSkillReadyImageAnalyzer.h" - -#include "Utils/NoWarningCV.h" - -#include -#include -#include - -#include - -#include "Config/OnnxSessions.h" -#include "Config/TaskData.h" -#include "Utils/Logger.hpp" - -bool asst::BattleSkillReadyImageAnalyzer::analyze() -{ - cv::Mat image = m_image(make_rect(m_roi)); - std::vector input = image_to_tensor(image); - - auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); - constexpr int64_t batch_size = 1; - std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; - - Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, input.data(), input.size(), - input_shape.data(), input_shape.size()); - - constexpr size_t classification_size = 2; - std::array results; - std::array output_shape { batch_size, classification_size }; - Ort::Value output_tensor = Ort::Value::CreateTensor(memory_info, results.data(), results.size(), - output_shape.data(), output_shape.size()); - - auto& session = OnnxSessions::get_instance().get("skill_ready_cls"); - // 这俩是hardcode在模型里的 - constexpr const char* input_names[] = { "input" }; // session.GetInputName() - constexpr const char* output_names[] = { "output" }; // session.GetOutputName() - - Ort::RunOptions run_options; - session.Run(run_options, input_names, &input_tensor, 1, output_names, &output_tensor, 1); - Log.info(__FUNCTION__, "raw result: ", results); - - softmax(results); - Log.info(__FUNCTION__, "after softmax: ", results); - bool ready = results[1] > results[0]; - -#ifdef ASST_DEBUG - if (ready) { - cv::rectangle(m_image_draw, make_rect(m_roi), cv::Scalar(0, 165, 255), 2); - cv::putText(m_image_draw, std::to_string(results[1]), cv::Point(m_roi.x, m_roi.y - 10), 1, 1.2, - cv::Scalar(0, 165, 255), 2); - } -#endif - - return ready; -} - -void asst::BattleSkillReadyImageAnalyzer::set_base_point(const Point& pt) -{ - auto task_ptr = Task.get("BattleSkillReady"); - const Rect& skill_roi_move = task_ptr->rect_move; - - set_roi(Rect(pt.x, pt.y, 0, 0).move(skill_roi_move)); -} diff --git a/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.h deleted file mode 100644 index 5d3143fdf7..0000000000 --- a/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "Vision/OnnxHelper.h" - -namespace asst -{ - class BattleSkillReadyImageAnalyzer final : public OnnxHelper - { - public: - using OnnxHelper::OnnxHelper; - virtual ~BattleSkillReadyImageAnalyzer() override = default; - - bool analyze(); - - void set_base_point(const Point& pt); - }; -} diff --git a/src/MaaCore/Vision/Battle/BattlefieldClassifier.cpp b/src/MaaCore/Vision/Battle/BattlefieldClassifier.cpp new file mode 100644 index 0000000000..b1188885d5 --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattlefieldClassifier.cpp @@ -0,0 +1,151 @@ +#include "BattlefieldClassifier.h" + +#include "Utils/NoWarningCV.h" + +#include +#include +#include + +#include + +#include "Config/OnnxSessions.h" +#include "Config/TaskData.h" +#include "Utils/Logger.hpp" + +using namespace asst; + +BattlefieldClassifier::ResultOpt BattlefieldClassifier::analyze() const +{ + Result result { .object_of_interest = m_object_of_interest }; + bool analyzed = false; + + if (m_object_of_interest.skill_ready) { + result.skill_ready = skill_ready_analyze(); + analyzed = true; + } + + if (m_object_of_interest.deploy_direction) { + result.deploy_direction = deploy_direction_analyze(); + analyzed = true; + } + + if (!analyzed) { + return std::nullopt; + } + + return result; +} + +BattlefieldClassifier::SkillReadyResult BattlefieldClassifier::skill_ready_analyze() const +{ + auto task_ptr = Task.get("BattleSkillReady"); + const Rect& skill_roi_move = task_ptr->rect_move; + Rect roi = Rect(m_base_point.x, m_base_point.y, 0, 0).move(skill_roi_move); + + cv::Mat image = make_roi(m_image, correct_rect(roi, m_image)); + std::vector input = image_to_tensor(image); + + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + constexpr int64_t batch_size = 1; + std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; + + Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, input.data(), input.size(), + input_shape.data(), input_shape.size()); + SkillReadyResult::Raw raw_results; + std::array output_shape { batch_size, SkillReadyResult::ClsSize }; + Ort::Value output_tensor = Ort::Value::CreateTensor(memory_info, raw_results.data(), raw_results.size(), + output_shape.data(), output_shape.size()); + + auto& session = OnnxSessions::get_instance().get("skill_ready_cls"); + // 这俩是hardcode在模型里的 + constexpr const char* input_names[] = { "input" }; // session.GetInputName() + constexpr const char* output_names[] = { "output" }; // session.GetOutputName() + + Ort::RunOptions run_options; + session.Run(run_options, input_names, &input_tensor, 1, output_names, &output_tensor, 1); + Log.info(__FUNCTION__, "raw results:", raw_results); + + SkillReadyResult::Prob prob = softmax(raw_results); + Log.info(__FUNCTION__, "prob:", prob); + bool ready = prob[1] > prob[0]; + float score = std::max(prob[0], prob[1]); + +#ifdef ASST_DEBUG + if (ready) { + cv::rectangle(m_image_draw, make_rect(roi), cv::Scalar(0, 165, 255), 2); + cv::putText(m_image_draw, std::to_string(score), cv::Point(roi.x, roi.y - 10), 1, 1.2, cv::Scalar(0, 165, 255), + 2); + } +#endif + + return SkillReadyResult { + .ready = ready, + .rect = roi, + .score = score, + .raw = raw_results, + .prob = prob, + .base_point = m_base_point, + }; +} + +BattlefieldClassifier::DeployDirectionResult BattlefieldClassifier::deploy_direction_analyze() const +{ + const auto& task_ptr = Task.get("BattleDeployDirectionRectMove"); + const Rect& roi_move = task_ptr->rect_move; + Rect roi = Rect(m_base_point.x, m_base_point.y, 0, 0).move(roi_move); + + cv::Mat image = make_roi(m_image, correct_rect(roi, m_image)); + std::vector input = image_to_tensor(image); + + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + constexpr int64_t batch_size = 1; + std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; + + Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, input.data(), input.size(), + input_shape.data(), input_shape.size()); + + DeployDirectionResult::Raw raw_results; + std::array output_shape { batch_size, DeployDirectionResult::ClsSize }; + Ort::Value output_tensor = Ort::Value::CreateTensor(memory_info, raw_results.data(), raw_results.size(), + output_shape.data(), output_shape.size()); + + auto& session = OnnxSessions::get_instance().get("deploy_direction_cls"); + // 这俩是hardcode在模型里的 + constexpr const char* input_names[] = { "input" }; // session.GetInputName() + constexpr const char* output_names[] = { "output" }; // session.GetOutputName() + + Ort::RunOptions run_options; + session.Run(run_options, input_names, &input_tensor, 1, output_names, &output_tensor, 1); + Log.info(__FUNCTION__, "raw result:", raw_results); + + DeployDirectionResult::Prob prob = softmax(raw_results); + Log.info(__FUNCTION__, "after softmax:", prob); + + size_t class_id = std::max_element(prob.begin(), prob.end()) - prob.begin(); + +#ifdef ASST_DEBUG + static const std::unordered_map ClassNames = { + { 0, "Right" }, + { 1, "Down" }, + { 2, "Left" }, + { 3, "Up" }, + }; + if (ClassNames.size() != prob.size()) { + Log.error("ClassNames.size() != prob.size()", ClassNames.size(), prob.size()); + throw std::runtime_error("ClassNames.size() != prob.size()"); + } + cv::putText(m_image_draw, ClassNames.at(class_id), cv::Point(roi.x, roi.y + roi.height), cv::FONT_HERSHEY_PLAIN, + 1.2, cv::Scalar(0, 255, 0), 2); + cv::putText(m_image_draw, std::to_string(prob[class_id]), cv::Point(roi.x, roi.y + roi.height + 20), + cv::FONT_HERSHEY_PLAIN, 1.2, cv::Scalar(0, 255, 0), 2); +#endif + + return DeployDirectionResult { + .direction = static_cast(class_id), + .rect = roi, + .score = prob[class_id], + .raw = raw_results, + .prob = prob, + .base_point = m_base_point, + }; +} diff --git a/src/MaaCore/Vision/Battle/BattlefieldClassifier.h b/src/MaaCore/Vision/Battle/BattlefieldClassifier.h new file mode 100644 index 0000000000..e722d2c797 --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattlefieldClassifier.h @@ -0,0 +1,75 @@ +#pragma once + +#include "Common/AsstBattleDef.h" +#include "Vision/OnnxHelper.h" +#include "Vision/VisionHelper.h" + +namespace asst +{ + class BattlefieldClassifier final : public VisionHelper, public OnnxHelper + { + public: + struct ObjectOfInterest + { + bool skill_ready = false; + bool deploy_direction = false; + }; + + public: + struct SkillReadyResult + { + static constexpr size_t ClsSize = 2; + using Raw = std::array; + using Prob = Raw; + + bool ready = false; + Rect rect; + float score = .0f; + Raw raw; + Prob prob; // after softmax + + Point base_point; + }; + + struct DeployDirectionResult + { + static constexpr size_t ClsSize = 4; + using Raw = std::array; + using Prob = Raw; + + battle::DeployDirection direction = battle::DeployDirection::None; + Rect rect; + float score = .0f; + Raw raw; + Prob prob; // after softmax + + Point base_point; + }; + + struct Result + { + ObjectOfInterest object_of_interest; + + SkillReadyResult skill_ready; + DeployDirectionResult deploy_direction; + }; + + using ResultOpt = std::optional; + + public: + using VisionHelper::VisionHelper; + virtual ~BattlefieldClassifier() override = default; + + void set_object_of_interest(ObjectOfInterest obj) { m_object_of_interest = obj; } + void set_base_point(const Point& pt) { m_base_point = pt; } + + ResultOpt analyze() const; + + protected: + SkillReadyResult skill_ready_analyze() const; + DeployDirectionResult deploy_direction_analyze() const; + + ObjectOfInterest m_object_of_interest; // 待识别的目标 + Point m_base_point; + }; +} // namespace asst diff --git a/src/MaaCore/Vision/Battle/BattlefieldDetector.cpp b/src/MaaCore/Vision/Battle/BattlefieldDetector.cpp new file mode 100644 index 0000000000..3facf67770 --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattlefieldDetector.cpp @@ -0,0 +1,146 @@ +#include "BattlefieldDetector.h" + +#include +#include +#include + +#include "Utils/NoWarningCV.h" +#include + +#include "Config/Miscellaneous/TilePack.h" +#include "Config/OnnxSessions.h" +#include "Config/TaskData.h" +#include "Utils/Logger.hpp" + +using namespace asst; + +BattlefieldDetector::ResultOpt BattlefieldDetector::analyze() const +{ + LogTraceFunction; + + Result result { .object_of_interest = m_object_of_interest }; + bool analyzed = false; + + if (m_object_of_interest.operators) { + result.operators = operator_analyze(); + analyzed = true; + } + + if (!analyzed) { + return std::nullopt; + } + + return result; +} + +std::vector BattlefieldDetector::operator_analyze() const +{ + const double x_scale = 640.0 / m_image.cols; + const double y_scale = 640.0 / m_image.rows; + + cv::Mat image; + cv::resize(m_image, image, cv::Size(), x_scale, y_scale, cv::INTER_AREA); + std::vector input = image_to_tensor(image); + + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + constexpr int64_t batch_size = 1; + std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; + + Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, input.data(), input.size(), + input_shape.data(), input_shape.size()); + + auto& session = OnnxSessions::get_instance().get("operators_det"); + + Ort::AllocatorWithDefaultOptions allocator; + std::string input_name = session.GetInputNameAllocated(0, allocator).get(); + std::string output_name = session.GetOutputNameAllocated(0, allocator).get(); + std::vector input_names = { input_name.c_str() }; + std::vector output_names = { output_name.c_str() }; + + Ort::RunOptions run_options; + auto output_tensors = session.Run(run_options, input_names.data(), &input_tensor, input_names.size(), + output_names.data(), output_names.size()); + + const float* raw_output = output_tensors[0].GetTensorData(); + // output_shape is { 1, 5, 8400 } + std::vector output_shape = output_tensors[0].GetTensorTypeAndShapeInfo().GetShape(); + + // yolov8 的 onnx 输出和前面的 v5, v7 等似乎不太一样，目前网上 yolov8 的 demo 较少，文档也没找到 + // 这里的输出解析是我跟着数据推测的： + // center_x0, center_x1, ..... center_x8399 + // center_y0, center_y1, ..... center_y8399 + // w0, w1, ..... w8399 + // h0, h1, ..... h8399 + // conf0, conf1, ..... conf8399 + // 如果后面要做多分类，可能得再看下怎么改（我也不知道shape会变成啥样） + std::vector> output(output_shape[1]); + for (int64_t i = 0; i < output_shape[1]; i++) { + output[i] = std::vector(raw_output + i * output_shape[2], raw_output + (i + 1) * output_shape[2]); + } + +#ifdef ASST_DEBUG + + const int draw_offset_y = static_cast(m_image.rows * -0.15); + const int draw_offset_h = static_cast(m_image.rows * 0.13); + +#endif + + std::vector all_results; + const auto& conf_vec = output.back(); + for (size_t i = 0; i < conf_vec.size(); ++i) { + float score = conf_vec[i]; + constexpr float Threshold = 0.3f; + if (score < Threshold) { + continue; + } + + int center_x = static_cast(output[0][i] / x_scale); + int center_y = static_cast(output[1][i] / y_scale); + int w = static_cast(output[2][i] / x_scale); + int h = static_cast(output[3][i] / y_scale); + + int x = center_x - w / 2; + int y = center_y - h / 2; + Rect rect { x, y, w, h }; + + OperatorResult box { OperatorResult::Cls::Operator, rect, score }; + +#ifdef ASST_DEBUG + + Rect draw_rect = box.rect; + draw_rect.y += draw_offset_y; + draw_rect.height += draw_offset_h; + cv::rectangle(m_image_draw, make_rect(draw_rect), cv::Scalar(0, 0, 255), 5); + cv::putText(m_image_draw, std::to_string(box.score), cv::Point(draw_rect.x, draw_rect.y - 10), + cv::FONT_HERSHEY_PLAIN, 1.2, cv::Scalar(0, 0, 255), 2); +#endif + + all_results.emplace_back(std::move(box)); + } + + // NMS + constexpr double NmsThreshold = 0.7f; + std::sort(all_results.begin(), all_results.end(), + [](const OperatorResult& a, const OperatorResult& b) { return a.score > b.score; }); + + std::vector nms_results; + for (size_t i = 0; i < all_results.size(); ++i) { + const auto& box = all_results[i]; + if (box.score < 0.1f) { + continue; + } + nms_results.emplace_back(box); + for (size_t j = i + 1; j < all_results.size(); ++j) { + auto& box2 = all_results[j]; + if (box2.score < 0.1f) { + continue; + } + int iou_area = (make_rect(box.rect) & make_rect(box2.rect)).area(); + if (iou_area > NmsThreshold * box2.rect.area()) { + box2.score = 0; + } + } + } + + return nms_results; +} diff --git a/src/MaaCore/Vision/Battle/BattlefieldDetector.h b/src/MaaCore/Vision/Battle/BattlefieldDetector.h new file mode 100644 index 0000000000..b4effee310 --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattlefieldDetector.h @@ -0,0 +1,51 @@ +#pragma once + +#include "Vision/OnnxHelper.h" + +namespace asst +{ + class BattlefieldDetector final : public VisionHelper, public OnnxHelper + { + public: + struct ObjectOfInterest + { + bool operators = false; + }; + + public: + // 战斗中干员（血条）检测，yolov8 模型 + struct OperatorResult + { + enum class Cls + { + Operator = 0, + }; + + Cls cls = Cls::Operator; + Rect rect; + float score = .0f; + }; + + struct Result + { + ObjectOfInterest object_of_interest; + + std::vector operators; + }; + + using ResultOpt = std::optional; + + public: + using VisionHelper::VisionHelper; + virtual ~BattlefieldDetector() override = default; + + void set_object_of_interest(ObjectOfInterest obj) { m_object_of_interest = obj; } + + ResultOpt analyze() const; + + protected: + std::vector operator_analyze() const; + + ObjectOfInterest m_object_of_interest; + }; +} diff --git a/src/MaaCore/Vision/OnnxHelper.cpp b/src/MaaCore/Vision/OnnxHelper.cpp index ce67697bf2..1e9c61dc05 100644 --- a/src/MaaCore/Vision/OnnxHelper.cpp +++ b/src/MaaCore/Vision/OnnxHelper.cpp @@ -2,7 +2,9 @@ #include "Utils/NoWarningCV.h" -std::vector asst::OnnxHelper::image_to_tensor(const cv::Mat& image) +using namespace asst; + +std::vector OnnxHelper::image_to_tensor(const cv::Mat& image) { cv::Mat src = image.clone(); cv::cvtColor(src, src, cv::COLOR_BGR2RGB); @@ -17,7 +19,7 @@ std::vector asst::OnnxHelper::image_to_tensor(const cv::Mat& image) return tensor; } -cv::Mat asst::OnnxHelper::hwc_to_chw(const cv::Mat& src) +cv::Mat OnnxHelper::hwc_to_chw(const cv::Mat& src) { std::vector rgb_images; cv::split(src, rgb_images); diff --git a/src/MaaCore/Vision/OnnxHelper.h b/src/MaaCore/Vision/OnnxHelper.h index 50e3829c2d..165e52166d 100644 --- a/src/MaaCore/Vision/OnnxHelper.h +++ b/src/MaaCore/Vision/OnnxHelper.h @@ -4,25 +4,26 @@ namespace asst { - class OnnxHelper : public VisionHelper + class OnnxHelper { public: - using VisionHelper::VisionHelper; - virtual ~OnnxHelper() override = default; + virtual ~OnnxHelper() = default; protected: template - inline static void softmax(T& input) + inline static T softmax(const T& input) { - float rowmax = *std::max_element(input.begin(), input.end()); - std::vector y(input.size()); + T output = input; + float rowmax = *std::max_element(output.begin(), output.end()); + std::vector y(output.size()); float sum = 0.0f; - for (size_t i = 0; i != input.size(); ++i) { - sum += y[i] = std::exp(input[i] - rowmax); + for (size_t i = 0; i != output.size(); ++i) { + sum += y[i] = std::exp(output[i] - rowmax); } - for (size_t i = 0; i != input.size(); ++i) { - input[i] = y[i] / sum; + for (size_t i = 0; i != output.size(); ++i) { + output[i] = y[i] / sum; } + return output; } static std::vector image_to_tensor(const cv::Mat& image);