From f26c4bc3b6a8dcd81f80e60dc881f3ea10b9d2a6 Mon Sep 17 00:00:00 2001 From: Horror Proton <107091537+horror-proton@users.noreply.github.com> Date: Sun, 5 Nov 2023 00:47:53 +0800 Subject: [PATCH 1/4] feat: ability to recognize long drop list --- .../Task/Fight/StageDropsTaskPlugin.cpp | 55 ++++++++++++++++++- .../Miscellaneous/StageDropsImageAnalyzer.cpp | 18 +++++- .../Miscellaneous/StageDropsImageAnalyzer.h | 3 +- 3 files changed, 70 insertions(+), 6 deletions(-) diff --git a/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp b/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp index 754f73557e..6e50b4e33c 100644 --- a/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp +++ b/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp @@ -4,6 +4,7 @@ #include #include +#include "Common/AsstTypes.h" #include "Common/AsstVersion.h" #include "Config/Miscellaneous/ItemConfig.h" #include "Config/Miscellaneous/StageDropsConfig.h" @@ -13,6 +14,7 @@ #include "Task/ProcessTask.h" #include "Task/ReportDataTask.h" #include "Utils/Logger.hpp" +#include "Vision/Matcher.h" #include "Vision/Miscellaneous/StageDropsImageAnalyzer.h" bool asst::StageDropsTaskPlugin::verify(AsstMsg msg, const json::value& details) const @@ -109,8 +111,57 @@ bool asst::StageDropsTaskPlugin::recognize_drops() return false; } - StageDropsImageAnalyzer analyzer(ctrler()->get_image()); - bool ret = analyzer.analyze(); + auto image_stride = ctrler()->get_image().clone(); + StageDropsImageAnalyzer analyzer(image_stride); + + bool ret = false; + Point cropped_out = {}; + while (!analyzer.analyze_baseline(cropped_out)) { + ret = false; + if (cropped_out == Point {}) break; // other error, return false + + // else: more materials to reveal? + + cropped_out.y -= 4; + cropped_out.x -= image_stride.cols - WindowWidthDefault; + const int swipe_dist = 200; + ctrler()->swipe(cropped_out, cropped_out + swipe_dist * Point::left(), 1000, true); + sleep(2000); + + const cv::Rect ref_roi = { image_stride.cols - 320, 520, 280, 120 }; + auto new_img = ctrler()->get_image(); + + Matcher offset_match(new_img(cv::Rect { 0, ref_roi.y, new_img.cols, ref_roi.height })); + offset_match.set_templ(image_stride(ref_roi)); + offset_match.set_threshold(0.7); + if (!offset_match.analyze()) break; + + // int offset = ref_roi.x - offset_match.get_result().rect.x; + const int offset = (new_img.cols - offset_match.get_result().rect.x) - (image_stride.cols - ref_roi.x); + Log.trace("new image offset:", offset); + if (offset <= 4) { + ret = true; + break; + } + const int rel_x = offset + image_stride.cols - new_img.cols; + + const cv::Rect overlay_rect = { 540, 500, 740, 220 }; + cv::Rect overlay_rect_on_stride = overlay_rect; + overlay_rect_on_stride.x += rel_x; + + cv::Mat new_stride = + cv::Mat { image_stride.rows, overlay_rect_on_stride.br().x, image_stride.type(), cv::Scalar(0) }; + image_stride.copyTo(new_stride(cv::Rect { 0, 0, image_stride.cols, image_stride.rows })); + new_img(overlay_rect).copyTo(new_stride(overlay_rect_on_stride)); + + image_stride = new_stride; + + ret = true; + analyzer.set_image(image_stride); + } + + // image stride constructed, start main step + ret &= analyzer.analyze(); auto&& [code, difficulty] = analyzer.get_stage_key(); m_stage_code = std::move(code); diff --git a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp index d5684eb3b8..b6f7b2ba9d 100644 --- a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp @@ -3,6 +3,7 @@ #include #include +#include "Common/AsstTypes.h" #include "Utils/Ranges.hpp" #include "Utils/NoWarningCV.h" @@ -226,7 +227,7 @@ bool asst::StageDropsImageAnalyzer::analyze_drops() { LogTraceFunction; - if (!analyze_baseline()) { + if (Point p {}; !analyze_baseline(p) && p == Point {}) { return false; } @@ -361,10 +362,12 @@ bool asst::StageDropsImageAnalyzer::analyze_drops_for_12() return !flag_analyzer.analyze(); } -bool asst::StageDropsImageAnalyzer::analyze_baseline() +bool asst::StageDropsImageAnalyzer::analyze_baseline(Point& cropped_out) { LogTraceFunction; + m_baseline.clear(); + auto task_ptr = Task.get("StageDrops-BaseLine"); cv::Mat preprocessed_roi; @@ -386,7 +389,9 @@ bool asst::StageDropsImageAnalyzer::analyze_baseline() cv::getStructuringElement(cv::MORPH_RECT, { 3, 1 })); // cropping after derivatives, dilation, and erosion - cv::cvtColor(preprocessed_roi(make_rect(task_ptr->roi)), preprocessed_roi, cv::COLOR_BGR2GRAY); + auto roi = make_rect(task_ptr->roi); + roi.width = WindowWidthDefault - roi.br().x + m_image.cols; // image may be wider than 1280 + cv::cvtColor(preprocessed_roi(roi), preprocessed_roi, cv::COLOR_BGR2GRAY); } cv::Mat preprocessed_bin; @@ -457,6 +462,13 @@ bool asst::StageDropsImageAnalyzer::analyze_baseline() Log.trace(__FUNCTION__, "baseline", key.to_string()); } + if (m_image.cols - (x_offset + bounding_rect.width) < 30 + max_spacing) { + cropped_out.x = x_offset + bounding_rect.width; + cropped_out.y = task_ptr->roi.y; + Log.trace("bounding_rect.right=", cropped_out.x, ", more materials to reveal?"); + return false; + } + return !m_baseline.empty(); } diff --git a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h index 15b0133bc0..e44648c538 100644 --- a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h +++ b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h @@ -23,12 +23,13 @@ namespace asst // > const auto& get_drops() const noexcept { return m_drops; } + bool analyze_baseline(Point& cropped_out); + protected: bool analyze_stage_code(); bool analyze_times(); bool analyze_stars(); bool analyze_difficulty(); - bool analyze_baseline(); bool analyze_drops(); // 落叶殇火 活动（异格夜刀）, act24side, 2023-03 bool analyze_drops_for_CF(); From 69c6439d4c7dfb935e6ea3f34506c971b59e8ccd Mon Sep 17 00:00:00 2001 From: Horror Proton <107091537+horror-proton@users.noreply.github.com> Date: Sun, 5 Nov 2023 23:05:57 +0800 Subject: [PATCH 2/4] chore: revert some changes on analyze_baseline --- .../Task/Fight/StageDropsTaskPlugin.cpp | 19 +++++++++---------- .../Miscellaneous/StageDropsImageAnalyzer.cpp | 12 ++++-------- .../Miscellaneous/StageDropsImageAnalyzer.h | 3 +-- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp b/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp index 6e50b4e33c..8d1f4b2797 100644 --- a/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp +++ b/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp @@ -6,6 +6,7 @@ #include "Common/AsstTypes.h" #include "Common/AsstVersion.h" +#include "Config/GeneralConfig.h" #include "Config/Miscellaneous/ItemConfig.h" #include "Config/Miscellaneous/StageDropsConfig.h" #include "Config/TaskData.h" @@ -115,28 +116,26 @@ bool asst::StageDropsTaskPlugin::recognize_drops() StageDropsImageAnalyzer analyzer(image_stride); bool ret = false; - Point cropped_out = {}; - while (!analyzer.analyze_baseline(cropped_out)) { + while (true) { ret = false; - if (cropped_out == Point {}) break; // other error, return false - // else: more materials to reveal? + // more materials to reveal? + + auto swipe_begin = Point { WindowWidthDefault - 40, 632 }; - cropped_out.y -= 4; - cropped_out.x -= image_stride.cols - WindowWidthDefault; const int swipe_dist = 200; - ctrler()->swipe(cropped_out, cropped_out + swipe_dist * Point::left(), 1000, true); - sleep(2000); + ctrler()->swipe(swipe_begin, swipe_begin + swipe_dist * Point::left(), 500, true, 2, 0); + sleep(Config.get_options().task_delay * 3); - const cv::Rect ref_roi = { image_stride.cols - 320, 520, 280, 120 }; + const cv::Rect ref_roi = { image_stride.cols - 320, 530, 280, 100 }; auto new_img = ctrler()->get_image(); + // find relative position of old image on new one Matcher offset_match(new_img(cv::Rect { 0, ref_roi.y, new_img.cols, ref_roi.height })); offset_match.set_templ(image_stride(ref_roi)); offset_match.set_threshold(0.7); if (!offset_match.analyze()) break; - // int offset = ref_roi.x - offset_match.get_result().rect.x; const int offset = (new_img.cols - offset_match.get_result().rect.x) - (image_stride.cols - ref_roi.x); Log.trace("new image offset:", offset); if (offset <= 4) { diff --git a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp index b6f7b2ba9d..d4b73706a1 100644 --- a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp @@ -3,7 +3,6 @@ #include #include -#include "Common/AsstTypes.h" #include "Utils/Ranges.hpp" #include "Utils/NoWarningCV.h" @@ -227,7 +226,7 @@ bool asst::StageDropsImageAnalyzer::analyze_drops() { LogTraceFunction; - if (Point p {}; !analyze_baseline(p) && p == Point {}) { + if (!analyze_baseline()) { return false; } @@ -362,7 +361,7 @@ bool asst::StageDropsImageAnalyzer::analyze_drops_for_12() return !flag_analyzer.analyze(); } -bool asst::StageDropsImageAnalyzer::analyze_baseline(Point& cropped_out) +bool asst::StageDropsImageAnalyzer::analyze_baseline() { LogTraceFunction; @@ -463,11 +462,8 @@ bool asst::StageDropsImageAnalyzer::analyze_baseline(Point& cropped_out) } if (m_image.cols - (x_offset + bounding_rect.width) < 30 + max_spacing) { - cropped_out.x = x_offset + bounding_rect.width; - cropped_out.y = task_ptr->roi.y; - Log.trace("bounding_rect.right=", cropped_out.x, ", more materials to reveal?"); - return false; - } + Log.trace("bounding_rect.right=", x_offset + bounding_rect.width, ", more materials to reveal?"); + } // TODO: else tell caller to prevent unnecessary swipe return !m_baseline.empty(); } diff --git a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h index e44648c538..15b0133bc0 100644 --- a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h +++ b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h @@ -23,13 +23,12 @@ namespace asst // > const auto& get_drops() const noexcept { return m_drops; } - bool analyze_baseline(Point& cropped_out); - protected: bool analyze_stage_code(); bool analyze_times(); bool analyze_stars(); bool analyze_difficulty(); + bool analyze_baseline(); bool analyze_drops(); // 落叶殇火 活动（异格夜刀）, act24side, 2023-03 bool analyze_drops_for_CF(); From 3cff6bebc950b6191fb86440c12bf7b8c946a1da Mon Sep 17 00:00:00 2001 From: Horror Proton <107091537+horror-proton@users.noreply.github.com> Date: Mon, 6 Nov 2023 14:00:16 +0800 Subject: [PATCH 3/4] chore: update variable name remove dead code, etc --- .../Task/Fight/StageDropsTaskPlugin.cpp | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp b/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp index 8d1f4b2797..27aa70350c 100644 --- a/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp +++ b/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp @@ -112,11 +112,11 @@ bool asst::StageDropsTaskPlugin::recognize_drops() return false; } - auto image_stride = ctrler()->get_image().clone(); - StageDropsImageAnalyzer analyzer(image_stride); + auto image_strip = ctrler()->get_image().clone(); + StageDropsImageAnalyzer analyzer(image_strip); bool ret = false; - while (true) { + while (!need_exit()) { ret = false; // more materials to reveal? @@ -127,39 +127,38 @@ bool asst::StageDropsTaskPlugin::recognize_drops() ctrler()->swipe(swipe_begin, swipe_begin + swipe_dist * Point::left(), 500, true, 2, 0); sleep(Config.get_options().task_delay * 3); - const cv::Rect ref_roi = { image_stride.cols - 320, 530, 280, 100 }; + const cv::Rect ref_roi = { image_strip.cols - 320, 530, 280, 100 }; auto new_img = ctrler()->get_image(); // find relative position of old image on new one Matcher offset_match(new_img(cv::Rect { 0, ref_roi.y, new_img.cols, ref_roi.height })); - offset_match.set_templ(image_stride(ref_roi)); + offset_match.set_templ(image_strip(ref_roi)); offset_match.set_threshold(0.7); if (!offset_match.analyze()) break; - const int offset = (new_img.cols - offset_match.get_result().rect.x) - (image_stride.cols - ref_roi.x); + const int offset = (new_img.cols - offset_match.get_result().rect.x) - (image_strip.cols - ref_roi.x); Log.trace("new image offset:", offset); if (offset <= 4) { ret = true; break; } - const int rel_x = offset + image_stride.cols - new_img.cols; + const int rel_x = offset + image_strip.cols - new_img.cols; const cv::Rect overlay_rect = { 540, 500, 740, 220 }; - cv::Rect overlay_rect_on_stride = overlay_rect; - overlay_rect_on_stride.x += rel_x; + cv::Rect overlay_rect_on_strip = overlay_rect; + overlay_rect_on_strip.x += rel_x; - cv::Mat new_stride = - cv::Mat { image_stride.rows, overlay_rect_on_stride.br().x, image_stride.type(), cv::Scalar(0) }; - image_stride.copyTo(new_stride(cv::Rect { 0, 0, image_stride.cols, image_stride.rows })); - new_img(overlay_rect).copyTo(new_stride(overlay_rect_on_stride)); + cv::Mat new_strip = + cv::Mat { image_strip.rows, overlay_rect_on_strip.br().x, image_strip.type(), cv::Scalar(0) }; + image_strip.copyTo(new_strip(cv::Rect { 0, 0, image_strip.cols, image_strip.rows })); + new_img(overlay_rect).copyTo(new_strip(overlay_rect_on_strip)); - image_stride = new_stride; + image_strip = new_strip; - ret = true; - analyzer.set_image(image_stride); + analyzer.set_image(image_strip); } - // image stride constructed, start main step + // image strip constructed, start main step ret &= analyzer.analyze(); auto&& [code, difficulty] = analyzer.get_stage_key(); From cae0f2227a6fb0d01f9e454c2d157f7807f62f98 Mon Sep 17 00:00:00 2001 From: Horror Proton <107091537+horror-proton@users.noreply.github.com> Date: Wed, 8 Nov 2023 18:46:33 +0800 Subject: [PATCH 4/4] chore: move image merging to StageDrop analyzer --- .../Task/Fight/StageDropsTaskPlugin.cpp | 28 +++-------------- .../Miscellaneous/StageDropsImageAnalyzer.cpp | 30 +++++++++++++++++++ .../Miscellaneous/StageDropsImageAnalyzer.h | 5 ++++ 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp b/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp index 27aa70350c..7c50dc70f8 100644 --- a/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp +++ b/src/MaaCore/Task/Fight/StageDropsTaskPlugin.cpp @@ -112,8 +112,7 @@ bool asst::StageDropsTaskPlugin::recognize_drops() return false; } - auto image_strip = ctrler()->get_image().clone(); - StageDropsImageAnalyzer analyzer(image_strip); + StageDropsImageAnalyzer analyzer(ctrler()->get_image()); bool ret = false; while (!need_exit()) { @@ -127,35 +126,16 @@ bool asst::StageDropsTaskPlugin::recognize_drops() ctrler()->swipe(swipe_begin, swipe_begin + swipe_dist * Point::left(), 500, true, 2, 0); sleep(Config.get_options().task_delay * 3); - const cv::Rect ref_roi = { image_strip.cols - 320, 530, 280, 100 }; auto new_img = ctrler()->get_image(); - // find relative position of old image on new one - Matcher offset_match(new_img(cv::Rect { 0, ref_roi.y, new_img.cols, ref_roi.height })); - offset_match.set_templ(image_strip(ref_roi)); - offset_match.set_threshold(0.7); - if (!offset_match.analyze()) break; - - const int offset = (new_img.cols - offset_match.get_result().rect.x) - (image_strip.cols - ref_roi.x); + const auto offset_opt = analyzer.merge_image(new_img); + if (!offset_opt.has_value()) break; + const auto offset = offset_opt.value(); Log.trace("new image offset:", offset); if (offset <= 4) { ret = true; break; } - const int rel_x = offset + image_strip.cols - new_img.cols; - - const cv::Rect overlay_rect = { 540, 500, 740, 220 }; - cv::Rect overlay_rect_on_strip = overlay_rect; - overlay_rect_on_strip.x += rel_x; - - cv::Mat new_strip = - cv::Mat { image_strip.rows, overlay_rect_on_strip.br().x, image_strip.type(), cv::Scalar(0) }; - image_strip.copyTo(new_strip(cv::Rect { 0, 0, image_strip.cols, image_strip.rows })); - new_img(overlay_rect).copyTo(new_strip(overlay_rect_on_strip)); - - image_strip = new_strip; - - analyzer.set_image(image_strip); } // image strip constructed, start main step diff --git a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp index d4b73706a1..3120ea627f 100644 --- a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.cpp @@ -1,6 +1,7 @@ #include "StageDropsImageAnalyzer.h" #include +#include #include #include "Utils/Ranges.hpp" @@ -361,6 +362,35 @@ bool asst::StageDropsImageAnalyzer::analyze_drops_for_12() return !flag_analyzer.analyze(); } +std::optional asst::StageDropsImageAnalyzer::merge_image(const cv::Mat& new_img) +{ + LogTraceFunction; + + const cv::Rect ref_roi = { m_image.cols - 320, 530, 280, 100 }; + Matcher offset_match(new_img(cv::Rect { 0, ref_roi.y, new_img.cols, ref_roi.height })); + offset_match.set_templ(m_image(ref_roi)); + offset_match.set_threshold(0.7); + if (!offset_match.analyze()) { + Log.error("Unable to merge images"); + return std::nullopt; + } + const int offset = (new_img.cols - offset_match.get_result().rect.x) - (m_image.cols - ref_roi.x); + + const int rel_x = offset + m_image.cols - new_img.cols; + + const cv::Rect overlay_rect = { 540, 500, 740, 220 }; + cv::Rect overlay_rect_on_strip = overlay_rect; + overlay_rect_on_strip.x += rel_x; + + cv::Mat new_strip = cv::Mat { m_image.rows, overlay_rect_on_strip.br().x, m_image.type(), cv::Scalar(0) }; + m_image.copyTo(new_strip(cv::Rect { 0, 0, m_image.cols, m_image.rows })); + new_img(overlay_rect).copyTo(new_strip(overlay_rect_on_strip)); + + m_image = new_strip; + + return offset; +} + bool asst::StageDropsImageAnalyzer::analyze_baseline() { LogTraceFunction; diff --git a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h index 15b0133bc0..322f891032 100644 --- a/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h +++ b/src/MaaCore/Vision/Miscellaneous/StageDropsImageAnalyzer.h @@ -23,6 +23,11 @@ namespace asst // > const auto& get_drops() const noexcept { return m_drops; } + // merge a new image with a different material position into m_image + // might resize m_image. + // return offset in pixels + std::optional merge_image(const cv::Mat& new_img); + protected: bool analyze_stage_code(); bool analyze_times();