From 42762bfbb759f30e230669f37acdd263b4ebe859 Mon Sep 17 00:00:00 2001 From: MistEO Date: Sun, 1 Oct 2023 13:54:56 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=A4=9A=E6=A8=A1?= =?UTF-8?q?=E6=9D=BF=E5=8C=B9=E9=85=8D=EF=BC=8C=E4=BB=BB=E4=B8=80=E8=BE=BE?= =?UTF-8?q?=E5=88=B0=E9=98=88=E5=80=BC=E5=8D=B3=E8=A7=86=E4=B8=BA=E5=91=BD?= =?UTF-8?q?=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MultiMatcher 暂不支持，仅使用第一个模板 --- src/MaaCore/Common/AsstTypes.h | 6 +- src/MaaCore/Config/TaskData.cpp | 49 ++++-- src/MaaCore/Task/BattleHelper.cpp | 8 +- .../CombatRecordRecognitionTask.cpp | 2 +- src/MaaCore/Vision/Config/MatcherConfig.cpp | 18 ++- src/MaaCore/Vision/Config/MatcherConfig.h | 6 +- .../Infrast/InfrastOperImageAnalyzer.cpp | 6 +- src/MaaCore/Vision/Matcher.cpp | 152 ++++++++++-------- src/MaaCore/Vision/Matcher.h | 2 +- .../Vision/Miscellaneous/PipelineAnalyzer.cpp | 4 +- .../Miscellaneous/StageDropsImageAnalyzer.cpp | 4 +- src/MaaCore/Vision/MultiMatcher.cpp | 6 +- .../RoguelikeRecruitSupportAnalyzer.cpp | 2 +- 13 files changed, 160 insertions(+), 105 deletions(-) diff --git a/src/MaaCore/Common/AsstTypes.h b/src/MaaCore/Common/AsstTypes.h index 517b7ab876..24df4d0608 100644 --- a/src/MaaCore/Common/AsstTypes.h +++ b/src/MaaCore/Common/AsstTypes.h @@ -397,9 +397,9 @@ namespace asst MatchTaskInfo(MatchTaskInfo&&) noexcept = default; MatchTaskInfo& operator=(const MatchTaskInfo&) = default; MatchTaskInfo& operator=(MatchTaskInfo&&) noexcept = default; - std::string templ_name; // 匹配模板图片文件名 - double templ_threshold = 0; // 模板匹配阈值 - std::pair mask_range; // 掩码的二值化范围 + std::vector templ_names; // 匹配模板图片文件名 + std::vector templ_thresholds; // 模板匹配阈值 + std::pair mask_range; // 掩码的二值化范围 }; // hash 计算任务的信息 diff --git a/src/MaaCore/Config/TaskData.cpp b/src/MaaCore/Config/TaskData.cpp index 5fb3d114ee..e7fc3e7bc7 100644 --- a/src/MaaCore/Config/TaskData.cpp +++ b/src/MaaCore/Config/TaskData.cpp @@ -724,17 +724,46 @@ asst::TaskData::taskptr_t asst::TaskData::generate_match_task_info(std::string_v default_ptr = default_match_task_info_ptr; } auto match_task_info_ptr = std::make_shared(); -#ifdef ASST_DEBUG - if (task_json.get("template", "") == std::string(name) + ".png") { - Log.warn("template name of task", name, "could be omitted."); + auto templ_opt = task_json.find("template"); + if (!templ_opt) { + match_task_info_ptr->templ_names = { std::string(name) + ".png" }; } -#endif - // template 留空时不从模板任务继承 - match_task_info_ptr->templ_name = task_json.get("template", std::string(name) + ".png"); - m_templ_required.emplace(match_task_info_ptr->templ_name); + else if (templ_opt->is_string()) { + match_task_info_ptr->templ_names = { templ_opt->as_string() + ".png" }; + } + else if (templ_opt->is_array()) { + match_task_info_ptr->templ_names = to_string_list(templ_opt->as_array()); + } + else { + Log.error("Invalid template type in task", name); + return nullptr; + } + + m_templ_required.insert(match_task_info_ptr->templ_names.begin(), match_task_info_ptr->templ_names.end()); // 其余若留空则继承模板任务 - match_task_info_ptr->templ_threshold = task_json.get("templThreshold", default_ptr->templ_threshold); + + auto threshold_opt = task_json.find("threshold"); + if (!threshold_opt) { + match_task_info_ptr->templ_thresholds = default_ptr->templ_thresholds; + } + else if (threshold_opt->is_number()) { + match_task_info_ptr->templ_thresholds = { threshold_opt->as_double() }; + } + else if (threshold_opt->is_array()) { + ranges::copy(threshold_opt->as_array() | views::transform(&ranges::range_value_t::as_double), + std::back_inserter(match_task_info_ptr->templ_thresholds)); + } + else { + Log.error("Invalid threshold type in task", name); + return nullptr; + } + + if (match_task_info_ptr->templ_names.size() != match_task_info_ptr->templ_thresholds.size()) { + Log.error("Template count and threshold count not match in task", name); + return nullptr; + } + if (auto opt = task_json.find("maskRange")) { auto& mask_range = *opt; match_task_info_ptr->mask_range = @@ -892,8 +921,8 @@ bool asst::TaskData::append_base_task_info(taskptr_t task_info_ptr, std::string_ std::shared_ptr asst::TaskData::_default_match_task_info() { auto match_task_info_ptr = std::make_shared(); - match_task_info_ptr->templ_name = "__INVALID__"; - match_task_info_ptr->templ_threshold = TemplThresholdDefault; + match_task_info_ptr->templ_names = { "__INVALID__" }; + match_task_info_ptr->templ_thresholds = { TemplThresholdDefault }; return match_task_info_ptr; } diff --git a/src/MaaCore/Task/BattleHelper.cpp b/src/MaaCore/Task/BattleHelper.cpp index 692f557573..3e2b9a451a 100644 --- a/src/MaaCore/Task/BattleHelper.cpp +++ b/src/MaaCore/Task/BattleHelper.cpp @@ -132,14 +132,16 @@ bool asst::BattleHelper::update_deployment(bool init, const cv::Mat& reusable) BestMatcher avatar_analyzer(oper.avatar); if (oper.cooling) { Log.trace("start matching cooling", oper.index); - static const double cooling_threshold = Task.get("BattleAvatarCoolingData")->templ_threshold; + static const double cooling_threshold = + Task.get("BattleAvatarCoolingData")->templ_thresholds.front(); static const auto cooling_mask_range = Task.get("BattleAvatarCoolingData")->mask_range; avatar_analyzer.set_threshold(cooling_threshold); avatar_analyzer.set_mask_range(cooling_mask_range.first, cooling_mask_range.second, true, true); } else { - static const double threshold = Task.get("BattleAvatarData")->templ_threshold; - static const double drone_threshold = Task.get("BattleDroneAvatarData")->templ_threshold; + static const double threshold = Task.get("BattleAvatarData")->templ_thresholds.front(); + static const double drone_threshold = + Task.get("BattleDroneAvatarData")->templ_thresholds.front(); avatar_analyzer.set_threshold(oper.role == Role::Drone ? drone_threshold : threshold); } diff --git a/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp b/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp index e8931e6eb6..5ec6fda9eb 100644 --- a/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp +++ b/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp @@ -781,7 +781,7 @@ void asst::CombatRecordRecognitionTask::ananlyze_deployment_names(ClipInfo& clip continue; } BestMatcher avatar_analyzer(oper.avatar); - static const double threshold = Task.get("BattleAvatarDataForVideo")->templ_threshold; + static const double threshold = Task.get("BattleAvatarDataForVideo")->templ_thresholds.front(); avatar_analyzer.set_threshold(threshold); // static const double drone_threshold = Task.get("BattleDroneAvatarData")->templ_threshold; // avatar_analyzer.set_threshold(oper.role == battle::Role::Drone ? drone_threshold : threshold); diff --git a/src/MaaCore/Vision/Config/MatcherConfig.cpp b/src/MaaCore/Vision/Config/MatcherConfig.cpp index 9f9896d0e1..d5d384b81a 100644 --- a/src/MaaCore/Vision/Config/MatcherConfig.cpp +++ b/src/MaaCore/Vision/Config/MatcherConfig.cpp @@ -21,12 +21,22 @@ void MatcherConfig::set_task_info(const std::string& task_name) void MatcherConfig::set_templ(std::variant templ) { - m_params.templ = std::move(templ); + m_params.templs = { std::move(templ) }; } +// void asst::MatcherConfig::set_templ(std::vector> templs) +//{ +// m_params.templs = std::move(templs); +// } + void MatcherConfig::set_threshold(double templ_thres) noexcept { - m_params.templ_thres = templ_thres; + m_params.templ_thres = { templ_thres }; +} + +void asst::MatcherConfig::set_threshold(std::vector templ_thres) noexcept +{ + m_params.templ_thres = std::move(templ_thres); } void MatcherConfig::set_mask_range(int lower, int upper, bool mask_with_src, bool mask_with_close) @@ -38,8 +48,8 @@ void MatcherConfig::set_mask_range(int lower, int upper, bool mask_with_src, boo void MatcherConfig::_set_task_info(MatchTaskInfo task_info) { - m_params.templ = std::move(task_info.templ_name); - m_params.templ_thres = task_info.templ_threshold; + ranges::copy(task_info.templ_names, std::back_inserter(m_params.templs)); + m_params.templ_thres = std::move(task_info.templ_thresholds); m_params.mask_range = std::move(task_info.mask_range); _set_roi(task_info.roi); diff --git a/src/MaaCore/Vision/Config/MatcherConfig.h b/src/MaaCore/Vision/Config/MatcherConfig.h index 02c606420b..e735e7214f 100644 --- a/src/MaaCore/Vision/Config/MatcherConfig.h +++ b/src/MaaCore/Vision/Config/MatcherConfig.h @@ -12,8 +12,8 @@ namespace asst public: struct Params { - std::variant templ; - double templ_thres = 0.0; + std::vector> templs; + std::vector templ_thres; std::pair mask_range; bool mask_with_src = false; bool mask_with_close = false; @@ -29,7 +29,9 @@ namespace asst void set_task_info(const std::string& task_name); void set_templ(std::variant templ); + // void set_templ(std::vector> templs); void set_threshold(double templ_thres) noexcept; + void set_threshold(std::vector templ_thres) noexcept; void set_mask_range(int lower, int upper, bool mask_with_src = false, bool mask_with_close = false); protected: diff --git a/src/MaaCore/Vision/Infrast/InfrastOperImageAnalyzer.cpp b/src/MaaCore/Vision/Infrast/InfrastOperImageAnalyzer.cpp index 7192927015..0fd8e3a39a 100644 --- a/src/MaaCore/Vision/Infrast/InfrastOperImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Infrast/InfrastOperImageAnalyzer.cpp @@ -132,7 +132,7 @@ void asst::InfrastOperImageAnalyzer::mood_analyze() LogTraceFunction; const auto prg_task_ptr = Task.get("InfrastOperMoodProgressBar"); - uint8_t prg_lower_limit = static_cast(prg_task_ptr->templ_threshold); + uint8_t prg_lower_limit = static_cast(prg_task_ptr->templ_thresholds.front()); int prg_diff_thres = prg_task_ptr->special_params.front(); Rect rect_move = prg_task_ptr->rect_move; @@ -229,7 +229,7 @@ void asst::InfrastOperImageAnalyzer::skill_analyze() Matcher skill_analyzer(m_image); skill_analyzer.set_mask_range(task_ptr->mask_range.first, task_ptr->mask_range.second); - skill_analyzer.set_threshold(task_ptr->templ_threshold); + skill_analyzer.set_threshold(task_ptr->templ_thresholds.front()); for (auto&& oper : m_result) { Rect roi = task_ptr->rect_move; @@ -371,7 +371,7 @@ void asst::InfrastOperImageAnalyzer::selected_analyze() } } Log.trace("selected_analyze |", count); - oper.selected = count >= selected_task_ptr->templ_threshold; + oper.selected = count >= selected_task_ptr->templ_thresholds.front(); oper.rect = selected_rect; // 先凑合用（ } } diff --git a/src/MaaCore/Vision/Matcher.cpp b/src/MaaCore/Vision/Matcher.cpp index 147f30e555..916696d41e 100644 --- a/src/MaaCore/Vision/Matcher.cpp +++ b/src/MaaCore/Vision/Matcher.cpp @@ -11,79 +11,89 @@ using namespace asst; Matcher::ResultOpt Matcher::analyze() const { - const auto& [matched, templ, templ_name] = preproc_and_match(make_roi(m_image, m_roi), m_params); + const auto match_results = preproc_and_match(make_roi(m_image, m_roi), m_params); - if (matched.empty()) { - return std::nullopt; - } - - double min_val = 0.0, max_val = 0.0; - cv::Point min_loc, max_loc; - cv::minMaxLoc(matched, &min_val, &max_val, &min_loc, &max_loc); - - Rect rect(max_loc.x + m_roi.x, max_loc.y + m_roi.y, templ.cols, templ.rows); - if (std::isnan(max_val) || std::isinf(max_val)) { - max_val = 0; - } - if (m_log_tracing && max_val > m_params.templ_thres * 0.7) { // 得分太低的肯定不对，没必要打印 - Log.trace("match_templ |", templ_name, "score:", max_val, "rect:", rect, "roi:", m_roi); - } - - if (max_val < m_params.templ_thres) { - return std::nullopt; - } - - // FIXME: 老接口太难重构了，先弄个这玩意兼容下，后续慢慢全删掉 - m_result.rect = rect; - m_result.score = max_val; - return m_result; -} - -Matcher::RawResult Matcher::preproc_and_match(const cv::Mat& image, const MatcherConfig::Params& params) -{ - cv::Mat templ; - std::string templ_name; - - if (std::holds_alternative(params.templ)) { - templ_name = std::get(params.templ); - templ = TemplResource::get_instance().get_templ(templ_name); - } - else if (std::holds_alternative(params.templ)) { - templ = std::get(params.templ); - } - else { - Log.error("templ is none"); - } - - if (templ.empty()) { - Log.error("templ is empty!", templ_name); -#ifdef ASST_DEBUG - throw std::runtime_error("templ is empty: " + templ_name); -#else - return {}; -#endif - } - - if (templ.cols > image.cols || templ.rows > image.rows) { - Log.error("templ size is too large", templ_name, "image size:", image.cols, image.rows, - "templ size:", templ.cols, templ.rows); - return {}; - } - - cv::Mat matched; - if (params.mask_range.first == 0 && params.mask_range.second == 0) { - cv::matchTemplate(image, templ, matched, cv::TM_CCOEFF_NORMED); - } - else { - cv::Mat mask; - cv::cvtColor(params.mask_with_src ? image : templ, mask, cv::COLOR_BGR2GRAY); - cv::inRange(mask, params.mask_range.first, params.mask_range.second, mask); - if (params.mask_with_close) { - cv::Mat kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(3, 3)); - cv::morphologyEx(mask, mask, cv::MORPH_CLOSE, kernel); + for (size_t i = 0; i < match_results.size(); ++i) { + const auto& [matched, templ, templ_name] = match_results[i]; + if (matched.empty()) { + continue; } - cv::matchTemplate(image, templ, matched, cv::TM_CCOEFF_NORMED, mask); + + double min_val = 0.0, max_val = 0.0; + cv::Point min_loc, max_loc; + cv::minMaxLoc(matched, &min_val, &max_val, &min_loc, &max_loc); + + Rect rect(max_loc.x + m_roi.x, max_loc.y + m_roi.y, templ.cols, templ.rows); + if (std::isnan(max_val) || std::isinf(max_val)) { + max_val = 0; + } + if (m_log_tracing && max_val > 0.5) { // 得分太低的肯定不对，没必要打印 + Log.trace("match_templ |", templ_name, "score:", max_val, "rect:", rect, "roi:", m_roi); + } + + double threshold = m_params.templ_thres[i]; + if (max_val < threshold) { + continue; + } + + // FIXME: 老接口太难重构了，先弄个这玩意兼容下，后续慢慢全删掉 + m_result.rect = rect; + m_result.score = max_val; + return m_result; } - return RawResult { .matched = matched, .templ = templ, .templ_name = templ_name }; + return std::nullopt; +} + +std::vector Matcher::preproc_and_match(const cv::Mat& image, const MatcherConfig::Params& params) +{ + std::vector results; + for (auto& ptempl : params.templs) { + cv::Mat templ; + std::string templ_name; + + if (std::holds_alternative(ptempl)) { + templ_name = std::get(ptempl); + templ = TemplResource::get_instance().get_templ(templ_name); + } + else if (std::holds_alternative(ptempl)) { + templ = std::get(ptempl); + } + else { + Log.error("templ is none"); + } + + if (templ.empty()) { + Log.error("templ is empty!", templ_name); +#ifdef ASST_DEBUG + throw std::runtime_error("templ is empty: " + templ_name); +#else + return {}; +#endif + } + + if (templ.cols > image.cols || templ.rows > image.rows) { + Log.error("templ size is too large", templ_name, "image size:", image.cols, image.rows, + "templ size:", templ.cols, templ.rows); + return {}; + } + + cv::Mat matched; + if (params.mask_range.first == 0 && params.mask_range.second == 0) { + cv::matchTemplate(image, templ, matched, cv::TM_CCOEFF_NORMED); + } + else { + cv::Mat mask; + cv::cvtColor(params.mask_with_src ? image : templ, mask, cv::COLOR_BGR2GRAY); + cv::inRange(mask, params.mask_range.first, params.mask_range.second, mask); + if (params.mask_with_close) { + cv::Mat kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(3, 3)); + cv::morphologyEx(mask, mask, cv::MORPH_CLOSE, kernel); + } + cv::matchTemplate(image, templ, matched, cv::TM_CCOEFF_NORMED, mask); + } + + results.emplace_back(RawResult { .matched = matched, .templ = templ, .templ_name = templ_name }); + } + return results; } diff --git a/src/MaaCore/Vision/Matcher.h b/src/MaaCore/Vision/Matcher.h index bd7ed182c1..aa5654821f 100644 --- a/src/MaaCore/Vision/Matcher.h +++ b/src/MaaCore/Vision/Matcher.h @@ -26,7 +26,7 @@ namespace asst cv::Mat templ; std::string templ_name; }; - static RawResult preproc_and_match(const cv::Mat& image, const MatcherConfig::Params& params); + static std::vector preproc_and_match(const cv::Mat& image, const MatcherConfig::Params& params); protected: virtual void _set_roi(const Rect& roi) override { set_roi(roi); } diff --git a/src/MaaCore/Vision/Miscellaneous/PipelineAnalyzer.cpp b/src/MaaCore/Vision/Miscellaneous/PipelineAnalyzer.cpp index 7ef9412353..de59c545ba 100644 --- a/src/MaaCore/Vision/Miscellaneous/PipelineAnalyzer.cpp +++ b/src/MaaCore/Vision/Miscellaneous/PipelineAnalyzer.cpp @@ -53,8 +53,8 @@ Matcher::ResultOpt PipelineAnalyzer::match(const std::shared_ptr& task Matcher match_analyzer(m_image, m_roi); const auto match_task_ptr = std::dynamic_pointer_cast(task_ptr); - if (match_task_ptr->templ_threshold > 1.0) { - Log.info(match_task_ptr->name, "'s threshold is", match_task_ptr->templ_threshold, ", just skip"); + if (ranges::all_of(match_task_ptr->templ_thresholds, [](double t) { return t > 1.0; })) { + Log.info(match_task_ptr->name, "'s threshold is", match_task_ptr->templ_thresholds, ", just skip"); return std::nullopt; } match_analyzer.set_task_info(match_task_ptr); diff --git a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp index de970f4fba..b91249bbaf 100644 --- a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp @@ -337,7 +337,7 @@ bool asst::StageDropsImageAnalyzer::analyze_baseline() int y_offset = task_ptr->roi.y + bounding_rect.y; const int min_width = task_ptr->special_params.front(); - const int max_spacing = static_cast(task_ptr->templ_threshold); + const int max_spacing = static_cast(task_ptr->templ_thresholds.front()); int i_start = 0, i_end = bounding.cols - 1; bool in = true; // 是否正处在白线中 @@ -542,7 +542,7 @@ std::optional asst::StageDropsImageAnalyzer::match_quantity_stri cv::inRange(gray, task_ptr->mask_range.first, task_ptr->mask_range.second, bin); // split - const int max_spacing = static_cast(task_ptr->templ_threshold); + const int max_spacing = static_cast(task_ptr->templ_thresholds.front()); std::vector contours; int i_right = bin.cols - 1, i_left = 0; bool in = false; diff --git a/src/MaaCore/Vision/MultiMatcher.cpp b/src/MaaCore/Vision/MultiMatcher.cpp index 558d6b0271..a1ac881905 100644 --- a/src/MaaCore/Vision/MultiMatcher.cpp +++ b/src/MaaCore/Vision/MultiMatcher.cpp @@ -14,19 +14,21 @@ using namespace asst; MultiMatcher::ResultsVecOpt MultiMatcher::analyze() const { - const auto& [matched, templ, templ_name] = Matcher::preproc_and_match(make_roi(m_image, m_roi), m_params); + auto match_results = Matcher::preproc_and_match(make_roi(m_image, m_roi), m_params); + const auto& [matched, templ, templ_name] = match_results.front(); if (matched.empty()) { return std::nullopt; } std::vector results; + double threshold = m_params.templ_thres.front(); int min_distance = (std::min)(templ.cols, templ.rows) / 2; for (int i = 0; i != matched.rows; ++i) { for (int j = 0; j != matched.cols; ++j) { auto value = matched.at(i, j); - if (value < m_params.templ_thres || std::isnan(value) || std::isinf(value)) { + if (value < threshold || std::isnan(value) || std::isinf(value)) { continue; } diff --git a/src/MaaCore/Vision/Roguelike/RoguelikeRecruitSupportAnalyzer.cpp b/src/MaaCore/Vision/Roguelike/RoguelikeRecruitSupportAnalyzer.cpp index e61fc067a3..672ccd4b78 100644 --- a/src/MaaCore/Vision/Roguelike/RoguelikeRecruitSupportAnalyzer.cpp +++ b/src/MaaCore/Vision/Roguelike/RoguelikeRecruitSupportAnalyzer.cpp @@ -151,7 +151,7 @@ int asst::RoguelikeRecruitSupportAnalyzer::match_elite(const Rect& roi, const in auto task_ptr = Task.get(task_name); analyzer.set_task_info(task_ptr); analyzer.set_roi(roi); - analyzer.set_threshold(Task.get(task_name)->templ_threshold); + analyzer.set_threshold(Task.get(task_name)->templ_thresholds.front()); if (!analyzer.analyze()) { continue;