From 91c3b7294191d4ace954e150ffaed8282fd064ae Mon Sep 17 00:00:00 2001 From: Horror Proton <107091537+horror-proton@users.noreply.github.com> Date: Wed, 4 Jan 2023 20:29:11 +0800 Subject: [PATCH] perf: update `DepotImageAnalyzer`, use Fourier series and update quantity match --- resource/tasks.json | 35 +--- .../Miscellaneous/DepotImageAnalyzer.cpp | 193 +++++++----------- .../Vision/Miscellaneous/DepotImageAnalyzer.h | 5 +- 3 files changed, 91 insertions(+), 142 deletions(-) diff --git a/resource/tasks.json b/resource/tasks.json index 39bdb29291..019ded8dbb 100644 --- a/resource/tasks.json +++ b/resource/tasks.json @@ -6692,8 +6692,8 @@ "roi": [ 0, 0, - 1060, - 596 + 1093, + 614 ], "roi_Doc": "作为截图的缩放尺寸使用", "maskRange": [ @@ -6705,36 +6705,17 @@ "template": "empty.png", "roi": [ 0, - 90, - 219, - 150 - ] + 190, + 156, + 190 + ], + "roi_Doc": "center of item at 1280 * 720: [x_first (unknown), y_first, x_period, y_period]" }, "DepotQuantity": { "template": "empty.png", - "templThreshold": 5, - "templThreshold_Doc": "这里用来作为字间距阈值", - "specialParams": [ - 120 - ], - "specialParams_Doc": "背景色亮度值上限", - "roi": [ - 36, - 69, - 47, - 20 - ], - "maskRange_Doc": "拿来做H通道的范围", "maskRange": [ 0, - 180 - ], - "specificRect_Doc": "拿来做S、V通道的范围", - "specificRect": [ - 0, - 35, - 125, - 255 + 110 ] }, "DepotBegin": { diff --git a/src/MaaCore/Vision/Miscellaneous/DepotImageAnalyzer.cpp b/src/MaaCore/Vision/Miscellaneous/DepotImageAnalyzer.cpp index cf856ed3bd..5fcd2a4b86 100644 --- a/src/MaaCore/Vision/Miscellaneous/DepotImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Miscellaneous/DepotImageAnalyzer.cpp @@ -4,10 +4,13 @@ #include "Config/Miscellaneous/ItemConfig.h" #include "Config/TaskData.h" +#include "Config/TemplResource.h" #include "Utils/Logger.hpp" #include "Vision/MatchImageAnalyzer.h" #include "Vision/OcrWithPreprocessImageAnalyzer.h" +#include + bool asst::DepotImageAnalyzer::analyze() { LogTraceFunction; @@ -53,47 +56,56 @@ void asst::DepotImageAnalyzer::resize() #endif } +template +cv::Mat asst::DepotImageAnalyzer::image_from_function(const cv::Size& size, const F& func) +{ + auto result = cv::Mat1f { size, CV_32F }; + for (int i = 0; i < result.cols; ++i) + for (int j = 0; j < result.rows; ++j) { + result.at(j, i) = func(i, j); + } + return result; +} + bool asst::DepotImageAnalyzer::analyze_base_rect() { LogTraceFunction; - Rect base_roi = Task.get("DepotBaseRect")->roi; - ItemInfo base_item_info; - size_t pos = match_item(base_roi, base_item_info, 0ULL, false); - if (pos == NPos) { - return false; - } - const auto& base_rect = base_item_info.rect; - const auto& [base_x, base_y, base_w, base_h] = base_rect; -#ifdef ASST_DEBUG - cv::rectangle(m_image_draw_resized, make_rect(base_rect), cv::Scalar(0, 0, 255), 2); -#endif + static constexpr int tm_size = 183 * 720 / 1080; - // 每个材料之间有间隔，这里 高度 * 2 的 roi, 一定有一个完整的材料 + 一个不完整的材料。 - // 所以识别结果只会有一个（除非这个间隔非常大） - const Rect vertical_rect(base_x, base_y + base_h, base_w, base_h * 2); - ItemInfo vertical_item_info; - pos = match_item(vertical_rect, vertical_item_info, pos + 1); + auto task_ptr = Task.get("DepotBaseRect"); + int x_period = task_ptr->roi.width * m_image_resized.cols / WindowWidthDefault; + int y_first = task_ptr->roi.y * m_image_resized.rows / WindowHeightDefault; + int y_period = task_ptr->roi.height * m_image_resized.rows / WindowHeightDefault; -#ifdef ASST_DEBUG - cv::rectangle(m_image_draw_resized, make_rect(vertical_item_info.rect), cv::Scalar(0, 0, 255), 2); -#endif + cv::Mat hist_x; + cv::Mat resized_gray; + cv::cvtColor(m_image_resized, resized_gray, cv::COLOR_RGB2GRAY); + cv::reduce(resized_gray, hist_x, 0, cv::REDUCE_AVG, CV_32F); - // 宽度 * 2，同上 - const Rect horizontal_roi(base_x + base_w, base_y, base_w * 2, base_h); - ItemInfo horizontal_item_info; - pos = match_item(horizontal_roi, horizontal_item_info, pos + 1); + cv::Mat sine_image = image_from_function({ resized_gray.cols, 1 }, [=](int x, int) -> float { + return static_cast(std::sin(2. * std::numbers::pi_v * x / x_period)); + }); -#ifdef ASST_DEBUG - cv::rectangle(m_image_draw_resized, make_rect(horizontal_item_info.rect), cv::Scalar(0, 0, 255), 2); -#endif + cv::Mat cosine_image = image_from_function({ resized_gray.cols, 1 }, [=](int x, int) -> float { + return static_cast(std::cos(2. * std::numbers::pi_v * x / x_period)); + }); - const int horizontal_spacing = horizontal_item_info.rect.x - (base_x + base_w); - const int vertical_spacing = vertical_item_info.rect.y - (base_y + base_h); + cv::Mat hist_sine; + cv::Mat hist_cosine; + cv::multiply(hist_x, sine_image, hist_sine); + cv::multiply(hist_x, cosine_image, hist_cosine); - for (int x = base_x; x <= m_image_resized.cols; x += (base_w + horizontal_spacing)) { - for (int y = base_y; y <= m_image_resized.rows; y += (base_h + vertical_spacing)) { - Rect item_roi = Rect(x, y, base_w, base_h); + const double s_v = cv::sum(hist_sine)[0]; + const double c_v = cv::sum(hist_cosine)[0]; + + const double phase = std::atan2(s_v, c_v); + double x_first = phase / (2. * std::numbers::pi_v)*x_period + x_period / 2.; + if (phase < 0) x_first += x_period; + + for (int x = static_cast(x_first); x <= m_image_resized.cols; x += x_period) { + for (int y = y_first; y <= m_image_resized.rows; y += y_period) { + Rect item_roi = Rect(x - tm_size / 2, y - tm_size / 2, tm_size, tm_size); if (!m_resized_rect.include(item_roi)) { continue; } @@ -123,7 +135,7 @@ bool asst::DepotImageAnalyzer::analyze_all_items() std::string item_id = info.item_id; m_match_begin_pos = cur_pos + 1; - info.quantity = match_quantity(info.rect); + info.quantity = match_quantity(info); info.item_name = ItemData.get_item_name(item_id); #ifdef ASST_DEBUG cv::putText(m_image_draw_resized, item_id, cv::Point(roi.x, roi.y - 10), cv::FONT_HERSHEY_SIMPLEX, 0.5, @@ -174,7 +186,11 @@ size_t asst::DepotImageAnalyzer::match_item(const Rect& roi, /* out */ ItemInfo& size_t matched_index = NPos; for (size_t index = begin_index, extra_count = 0; index < all_items.size(); ++index) { const std::string& item_id = all_items.at(index); - analyzer.set_templ_name(item_id); + // analyzer.set_templ_name(item_id); + // TODO: too slow? find a way to set mask directly + cv::Mat templ = TemplResource::get_instance().get_templ(item_id).clone(); + templ(cv::Rect { templ.cols - 80, templ.rows - 50, 80, 50 }) = cv::Scalar { 0, 0, 0 }; + analyzer.set_templ(templ); if (!analyzer.analyze()) { continue; } @@ -199,94 +215,42 @@ size_t asst::DepotImageAnalyzer::match_item(const Rect& roi, /* out */ ItemInfo& return matched_index; } -int asst::DepotImageAnalyzer::match_quantity(const Rect& roi) +int asst::DepotImageAnalyzer::match_quantity(const ItemInfo& item) { auto task_ptr = Task.get("DepotQuantity"); + auto item_templ = TemplResource::get_instance().get_templ(item.item_id); + auto item_image = m_image_resized(make_rect(item.rect)); + cv::Mat quotient; + cv::divide(item_image + cv::Scalar { 1, 1, 1 }, // I've forgot why I should plus 1 here + item_templ + cv::Scalar { 1, 1, 1 }, // plus 1 to avoid divide by zero + quotient, 255); - Rect quantity_roi = roi.move(task_ptr->roi); - cv::Mat quantity_img = m_image_resized(make_rect(quantity_roi)); - cv::Mat hsv; - cv::cvtColor(quantity_img, hsv, cv::COLOR_BGR2HSV); + cv::Mat mask_r; + cv::Mat mask_g; + cv::Mat mask_b; + static constexpr int lb = 60; + static constexpr int ub = 140; + cv::inRange(quotient, cv::Scalar { lb, 0, 0 }, cv::Scalar { ub, 255, 255 }, mask_r); + cv::inRange(quotient, cv::Scalar { 0, lb, 0 }, cv::Scalar { 255, ub, 255 }, mask_g); + cv::inRange(quotient, cv::Scalar { 0, 0, lb }, cv::Scalar { 255, 255, ub }, mask_b); + cv::Mat mask; + cv::bitwise_or(mask_r, mask_g, mask); + cv::bitwise_or(mask, mask_b, mask); - const int h_lower = task_ptr->mask_range.first; - const int h_upper = task_ptr->mask_range.second; - const int s_lower = task_ptr->specific_rect.x; - const int s_upper = task_ptr->specific_rect.y; - const int v_lower = task_ptr->specific_rect.width; - const int v_upper = task_ptr->specific_rect.height; + cv::morphologyEx(mask, mask, cv::MORPH_OPEN, cv::getStructuringElement(cv::MORPH_RECT, { 4, 4 })); + auto mask_rect = cv::boundingRect(mask); + if (mask_rect.height < 18) mask_rect.height = 18; + // minus 1 to trim white pixels + Rect ocr_roi { item.rect.x + mask_rect.x, item.rect.y + mask_rect.y, mask_rect.width - 1, mask_rect.height - 1 }; - cv::Mat bin; - cv::inRange(hsv, cv::Scalar(h_lower, s_lower, v_lower), cv::Scalar(h_upper, s_upper, v_upper), bin); - - // split - const int max_spacing = static_cast(task_ptr->templ_threshold); - const int bg_v_upper = task_ptr->special_params.front(); - std::vector contours; - int i_right = bin.cols - 1, i_left = 0; - bool in = false; - int spacing = 0; - - for (int i = bin.cols - 1; i >= 0; --i) { - bool has_white = false; - for (int j = 0; j < bin.rows; ++j) { - if (bin.at(j, i)) { - has_white = true; - break; - } - } - if (in && !has_white) { // leave - i_left = i; - in = false; - spacing = 0; - contours.emplace_back(i_left, i_right + 1); // range 是前闭后开的 - } - else if (!in && has_white) { // enter - i_right = i; - in = true; - } - else if (!in) { - ++spacing; - uchar bg_top_v = hsv.at(0, i)[2]; - uchar bg_btm_v = hsv.at(bin.rows - 1, i)[2]; - if (i_left != 0 && (spacing > max_spacing || bg_top_v > bg_v_upper || bg_btm_v > bg_v_upper)) { - // filter out noise - break; - } - } - } - - cv::Rect y_bounding_rect; - for (int i = 0; i < contours.size(); ++i) { - if (y_bounding_rect.empty()) { - auto temp = cv::boundingRect(bin(cv::Range::all(), contours.at(i))); - if (temp.height > 10) { - y_bounding_rect = temp; - } - else { - continue; - } - } - auto left_elem_rect = cv::boundingRect(bin(cv::Range::all(), contours.back())); - if (std::abs(left_elem_rect.height - y_bounding_rect.height) > 2 || - std::abs(left_elem_rect.y - y_bounding_rect.y) > 1 || left_elem_rect.width > 12) { - contours.pop_back(); - } - else { - break; - } - } - if (contours.empty()) { - return 0; - } - int far_left = contours.back().start; - int far_right = contours.front().end; + cv::Mat ocr_img = m_image_resized.clone(); + cv::subtract(m_image_resized(make_rect(item.rect)), item_templ * 0.41, + ocr_img(make_rect(item.rect))); OcrWithPreprocessImageAnalyzer analyzer(m_image_resized); analyzer.set_task_info("NumberOcrReplace"); - Rect ocr_roi(quantity_roi.x + far_left, quantity_roi.y + y_bounding_rect.y, far_right - far_left, - y_bounding_rect.height); analyzer.set_roi(ocr_roi); - analyzer.set_expansion(1); + analyzer.set_expansion(0); analyzer.set_threshold(task_ptr->mask_range.first, task_ptr->mask_range.second); if (!analyzer.analyze()) { @@ -302,12 +266,13 @@ int asst::DepotImageAnalyzer::match_quantity(const Rect& roi) #endif std::string digit_str = result.text; + utils::string_trim(digit_str, std::function { [](char c) -> bool { return c != '.'; } }); int multiple = 1; if (size_t w_pos = digit_str.find("万"); w_pos != std::string::npos) { multiple = 10000; digit_str.erase(w_pos, digit_str.size()); } - else if (size_t k_pos = digit_str.find("K"); k_pos != std::string::npos) { + else if (size_t k_pos = digit_str.find('K'); k_pos != std::string::npos) { multiple = 1000; digit_str.erase(k_pos, digit_str.size()); } @@ -315,7 +280,7 @@ int asst::DepotImageAnalyzer::match_quantity(const Rect& roi) multiple = 100000000; digit_str.erase(e_pos, digit_str.size()); } - else if (size_t m_pos = digit_str.find("M"); m_pos != std::string::npos) { + else if (size_t m_pos = digit_str.find('M'); m_pos != std::string::npos) { multiple = 1000000; digit_str.erase(m_pos, digit_str.size()); } diff --git a/src/MaaCore/Vision/Miscellaneous/DepotImageAnalyzer.h b/src/MaaCore/Vision/Miscellaneous/DepotImageAnalyzer.h index 03195b824d..071dfb1889 100644 --- a/src/MaaCore/Vision/Miscellaneous/DepotImageAnalyzer.h +++ b/src/MaaCore/Vision/Miscellaneous/DepotImageAnalyzer.h @@ -33,9 +33,12 @@ namespace asst bool check_roi_empty(const Rect& roi); size_t match_item(const Rect& roi, /* out */ ItemInfo& item_info, size_t begin_index = 0ULL, bool with_enlarge = true); - int match_quantity(const Rect& roi); + int match_quantity(const ItemInfo& item); Rect resize_rect_to_raw_size(const Rect& rect); + template + static cv::Mat image_from_function(const cv::Size& size, const F& func); + size_t m_match_begin_pos = 0ULL; Rect m_resized_rect; cv::Mat m_image_resized;