From b9f196c3a15887dafbf31d5fe9fb874fd0bd7650 Mon Sep 17 00:00:00 2001
From: status102 <102887808+status102@users.noreply.github.com>
Date: Tue, 13 May 2025 18:24:16 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E7=89=B9=E5=BE=81=E5=8C=B9=E9=85=8D=20?=
=?UTF-8?q?(#10966)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* feat: 特征匹配
* chore: 测试内容
* refactor: 根据最新版本重新迁移, 移除多模板
* chore: Auto update by pre-commit hooks [skip changelog]
* chore: 测试移除
* fix: platform
* chore: docs
---------
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
src/MaaCore/Common/AsstTypes.h | 10 +
src/MaaCore/MaaCore.vcxproj | 4 +
src/MaaCore/MaaCore.vcxproj.filters | 12 +
.../Vision/Config/FeatureMatcherConfig.cpp | 1 +
.../Vision/Config/FeatureMatcherConfig.h | 64 +++++
src/MaaCore/Vision/FeatureMatcher.cpp | 226 ++++++++++++++++++
src/MaaCore/Vision/FeatureMatcher.h | 58 +++++
src/MaaCore/Vision/VisionHelper.cpp | 47 ++++
src/MaaCore/Vision/VisionHelper.h | 7 +
9 files changed, 429 insertions(+)
create mode 100644 src/MaaCore/Vision/Config/FeatureMatcherConfig.cpp
create mode 100644 src/MaaCore/Vision/Config/FeatureMatcherConfig.h
create mode 100644 src/MaaCore/Vision/FeatureMatcher.cpp
create mode 100644 src/MaaCore/Vision/FeatureMatcher.h
diff --git a/src/MaaCore/Common/AsstTypes.h b/src/MaaCore/Common/AsstTypes.h
index 05ad2256f0..2faa7079c4 100644
--- a/src/MaaCore/Common/AsstTypes.h
+++ b/src/MaaCore/Common/AsstTypes.h
@@ -250,6 +250,16 @@ struct MatchRect
double score = 0.0;
std::string templ_name;
};
+
+struct FeatureMatchRect
+{
+ std::string to_string() const { return "{ rect: " + rect.to_string() + ", count: " + std::to_string(count) + " }"; }
+
+ explicit operator std::string() const { return to_string(); }
+
+ Rect rect;
+ int count = 0;
+};
} // namespace asst
namespace std
diff --git a/src/MaaCore/MaaCore.vcxproj b/src/MaaCore/MaaCore.vcxproj
index 4768d90032..0346db2ab8 100644
--- a/src/MaaCore/MaaCore.vcxproj
+++ b/src/MaaCore/MaaCore.vcxproj
@@ -293,6 +293,8 @@
+
+
@@ -488,6 +490,8 @@
+
+
diff --git a/src/MaaCore/MaaCore.vcxproj.filters b/src/MaaCore/MaaCore.vcxproj.filters
index 754ee62d07..b95e37c1a0 100644
--- a/src/MaaCore/MaaCore.vcxproj.filters
+++ b/src/MaaCore/MaaCore.vcxproj.filters
@@ -1286,6 +1286,12 @@
Source\Task\Roguelike
+
+ Source\Vision
+
+
+ Source\Vision\Config
+
Source\Controller
@@ -1791,6 +1797,12 @@
Source\Resource\TaskData
+
+ Source\Vision
+
+
+ Source\Vision\Config
+
Source\Task\Infrast
diff --git a/src/MaaCore/Vision/Config/FeatureMatcherConfig.cpp b/src/MaaCore/Vision/Config/FeatureMatcherConfig.cpp
new file mode 100644
index 0000000000..90e3b3ca66
--- /dev/null
+++ b/src/MaaCore/Vision/Config/FeatureMatcherConfig.cpp
@@ -0,0 +1 @@
+#include "FeatureMatcherConfig.h"
diff --git a/src/MaaCore/Vision/Config/FeatureMatcherConfig.h b/src/MaaCore/Vision/Config/FeatureMatcherConfig.h
new file mode 100644
index 0000000000..e3e85ff540
--- /dev/null
+++ b/src/MaaCore/Vision/Config/FeatureMatcherConfig.h
@@ -0,0 +1,64 @@
+#pragma once
+#include "Common/AsstTypes.h"
+#include "Utils/NoWarningCVMat.h"
+#include
+
+namespace asst
+{
+class FeatureMatcherConfig
+{
+public:
+ enum class Detector
+ {
+ SIFT, // 计算复杂度高，具有尺度不变性、旋转不变性。效果最好。
+ SURF,
+ ORB, // 计算速度非常快，具有旋转不变性。但不具有尺度不变性。
+ BRISK, // 计算速度非常快，具有尺度不变性、旋转不变性。
+ KAZE, // 适用于2D和3D图像，具有尺度不变性、旋转不变性。
+ AKAZE, // 计算速度较快，具有尺度不变性、旋转不变性。
+ };
+
+ // enum class Matcher
+ //{
+ // FLANN,
+ // BRUTEFORCE,
+ // };
+
+ struct Params
+ {
+ std::variant templs;
+ bool green_mask = false;
+
+ Detector detector = Detector::SIFT;
+ // Matcher matcher = Matcher::FLANN;
+
+ double distance_ratio = 0.6; // KNN 匹配算法的距离比值，[0 - 1.0], 越大则匹配越宽松,更容易连线。默认0.6
+ int count = 4; // 匹配的特征点的数量要求（阈值），默认4
+ };
+
+public:
+ FeatureMatcherConfig() = default;
+ virtual ~FeatureMatcherConfig() = default;
+
+ void set_params(Params params) { m_params = std::move(params); }
+
+ // void set_task_info(const std::shared_ptr& task_ptr);
+ // void set_task_info(const std::string& task_name);
+
+ void set_templ(std::variant templ) { m_params.templs = { std::move(templ) }; }
+
+ void set_detector(Detector detector) noexcept { m_params.detector = detector; }
+
+ void set_distance_ratio(double distance_ratio) noexcept { m_params.distance_ratio = distance_ratio; }
+
+ void set_count(int count) noexcept { m_params.count = count; }
+
+protected:
+ virtual void _set_roi(const Rect& roi) = 0;
+
+ // void _set_task_info(MatchTaskInfo task_info);
+
+protected:
+ Params m_params;
+};
+}
diff --git a/src/MaaCore/Vision/FeatureMatcher.cpp b/src/MaaCore/Vision/FeatureMatcher.cpp
new file mode 100644
index 0000000000..04bcf1c8ea
--- /dev/null
+++ b/src/MaaCore/Vision/FeatureMatcher.cpp
@@ -0,0 +1,226 @@
+#include "FeatureMatcher.h"
+
+#include "Config/TemplResource.h"
+#include "Utils/Logger.hpp"
+#include "Utils/NoWarningCV.h"
+
+// MAA_SUPPRESS_CV_WARNINGS_BEGIN
+#include
+#include
+#ifdef MAA_VISION_HAS_XFEATURES2D
+#include
+#endif
+// MAA_SUPPRESS_CV_WARNINGS_END
+
+asst::FeatureMatcher::ResultsVecOpt asst::FeatureMatcher::analyze() const
+{
+ auto start_time = std::chrono::steady_clock::now();
+ const auto& image = m_image;
+
+ const auto& templ_ptr = m_params.templs;
+ cv::Mat templ;
+ std::string templ_name;
+
+ if (std::holds_alternative(templ_ptr)) {
+ templ_name = std::get(templ_ptr);
+ templ = TemplResource::get_instance().get_templ(templ_name);
+ }
+ else if (std::holds_alternative(templ_ptr)) {
+ templ = std::get(templ_ptr);
+ }
+ else {
+ Log.error("templ is none");
+ }
+
+ if (templ.empty()) {
+ Log.error("templ is empty!", templ_name);
+#ifdef ASST_DEBUG
+ throw std::runtime_error("templ is empty: " + templ_name);
+#else
+ return std::nullopt;
+#endif
+ }
+
+ if (templ.cols > image.cols || templ.rows > image.rows) {
+ LogError << "templ size is too large" << templ_name << "image size:" << image.cols << image.rows
+ << "templ size:" << templ.cols << templ.rows;
+ return std::nullopt;
+ }
+
+ auto [keypoints_1, descriptors_1] = detect(templ, create_mask(templ, m_params.green_mask));
+
+ auto results = feature_match(templ, keypoints_1, descriptors_1);
+ std::erase_if(results, [&](const auto& res) { return res.count < m_params.count; });
+ m_result = std::move(results);
+ auto cost = std::chrono::duration_cast(std::chrono::steady_clock::now() - start_time);
+
+ for (const auto& r : m_result) {
+ Log.debug("feature_match |", templ_name, "count:", r.count, "rect:", r.rect, "roi:", m_roi);
+#ifdef ASST_DEBUG
+ cv::rectangle(m_image_draw, make_rect(r.rect), cv::Scalar(0, 0, 255), 2);
+#endif
+ }
+ Log.trace("count:", m_result.size(), ", cost:", cost.count());
+ return m_result;
+}
+
+std::pair, cv::Mat>
+ asst::FeatureMatcher::detect(const cv::Mat& image, const cv::Mat& mask) const
+{
+ auto detector = create_detector();
+ if (!detector) {
+ LogError << "detector is empty";
+ return {};
+ }
+
+ std::vector keypoints;
+ cv::Mat descriptors;
+ detector->detectAndCompute(image, mask, keypoints, descriptors);
+
+ return std::make_pair(std::move(keypoints), std::move(descriptors));
+}
+
+asst::FeatureMatcher::ResultsVec asst::FeatureMatcher::feature_match(
+ const cv::Mat& templ,
+ const std::vector& keypoints_1,
+ const cv::Mat& descriptors_1) const
+{
+ auto [keypoints_2, descriptors_2] = detect(m_image, create_mask(m_image, make_rect(m_roi)));
+
+ auto match_points = match(descriptors_1, descriptors_2);
+
+ std::vector good_matches;
+ ResultsVec results = feature_postproc(match_points, keypoints_1, keypoints_2, templ.cols, templ.rows, good_matches);
+
+ /*
+ if (debug_draw_) {
+ auto draw = draw_result(templ, keypoints_1, keypoints_2, good_matches, results);
+ handle_draw(draw);
+ }
+ */
+ return results;
+}
+
+std::vector>
+ asst::FeatureMatcher::match(const cv::Mat& descriptors_1, const cv::Mat& descriptors_2) const
+{
+ if (descriptors_1.empty() || descriptors_2.empty()) {
+ LogWarn << "descriptors is empty";
+ return {};
+ }
+
+ auto matcher = create_matcher();
+ if (!matcher) {
+ LogError << "matcher is empty";
+ return {};
+ }
+
+ std::vector train_desc(1, descriptors_1);
+ matcher->add(train_desc);
+ matcher->train();
+
+ std::vector> match_points;
+ matcher->knnMatch(descriptors_2, match_points, 2);
+ return match_points;
+}
+
+asst::FeatureMatcher::ResultsVec asst::FeatureMatcher::feature_postproc(
+ const std::vector>& match_points,
+ const std::vector& keypoints_1,
+ const std::vector& keypoints_2,
+ int templ_cols,
+ int templ_rows,
+ std::vector& good_matches) const
+{
+ std::vector obj;
+ std::vector scene;
+
+ for (const auto& point : match_points) {
+ if (point.size() != 2) {
+ continue;
+ }
+
+ double threshold = m_params.distance_ratio * point[1].distance;
+ if (point[0].distance > threshold) {
+ continue;
+ }
+ good_matches.emplace_back(point[0]);
+ obj.emplace_back(keypoints_1[point[0].trainIdx].pt);
+ scene.emplace_back(keypoints_2[point[0].queryIdx].pt);
+ }
+
+ LogDebug << "Match:" << VAR(good_matches.size()) << VAR(match_points.size()) << VAR(m_params.distance_ratio);
+
+ if (good_matches.size() < 4) {
+ return {};
+ }
+
+ cv::Mat homography = cv::findHomography(obj, scene, cv::RHO);
+
+ if (homography.empty()) {
+ LogDebug << "Homography is empty";
+ return {};
+ }
+
+ std::array obj_corners = { cv::Point2d(0, 0),
+ cv::Point2d(templ_cols, 0),
+ cv::Point2d(templ_cols, templ_rows),
+ cv::Point2d(0, templ_rows) };
+ std::array scene_corners;
+ cv::perspectiveTransform(obj_corners, scene_corners, homography);
+
+ double x = std::min({ scene_corners[0].x, scene_corners[1].x, scene_corners[2].x, scene_corners[3].x });
+ double y = std::min({ scene_corners[0].y, scene_corners[1].y, scene_corners[2].y, scene_corners[3].y });
+ double w = std::max({ scene_corners[0].x, scene_corners[1].x, scene_corners[2].x, scene_corners[3].x }) - x;
+ double h = std::max({ scene_corners[0].y, scene_corners[1].y, scene_corners[2].y, scene_corners[3].y }) - y;
+ cv::Rect box { static_cast(x), static_cast(y), static_cast(w), static_cast(h) };
+ box &= make_rect(m_roi);
+
+ size_t count = std::ranges::count_if(scene, [&box](const auto& point) { return box.contains(point); });
+
+ return { Result { .rect = make_rect(box), .count = static_cast(count) } };
+}
+
+cv::Ptr asst::FeatureMatcher::create_detector() const
+{
+ switch (m_params.detector) {
+ case FeatureMatcherConfig::Detector::SIFT:
+ return cv::SIFT::create();
+ case FeatureMatcherConfig::Detector::ORB:
+ return cv::ORB::create();
+ case FeatureMatcherConfig::Detector::BRISK:
+ return cv::BRISK::create();
+ case FeatureMatcherConfig::Detector::KAZE:
+ return cv::KAZE::create();
+ case FeatureMatcherConfig::Detector::AKAZE:
+ return cv::AKAZE::create();
+ case FeatureMatcherConfig::Detector::SURF:
+#ifdef MAA_VISION_HAS_XFEATURES2D
+ return cv::xfeatures2d::SURF::create();
+#else
+ Log.error("SURF not enabled!");
+ return nullptr;
+#endif
+ }
+
+ Log.error("Unknown detector", static_cast(m_params.detector));
+ return nullptr;
+}
+
+cv::Ptr asst::FeatureMatcher::create_matcher() const
+{
+ switch (m_params.detector) {
+ case FeatureMatcherConfig::Detector::SIFT:
+ case FeatureMatcherConfig::Detector::SURF:
+ case FeatureMatcherConfig::Detector::KAZE:
+ return cv::FlannBasedMatcher::create();
+
+ case FeatureMatcherConfig::Detector::ORB:
+ case FeatureMatcherConfig::Detector::BRISK:
+ case FeatureMatcherConfig::Detector::AKAZE:
+ return cv::BFMatcher::create(cv::NORM_HAMMING);
+ }
+
+ Log.error("Unknown detector", static_cast(m_params.detector));
+ return nullptr;
+}
diff --git a/src/MaaCore/Vision/FeatureMatcher.h b/src/MaaCore/Vision/FeatureMatcher.h
new file mode 100644
index 0000000000..a4b479c776
--- /dev/null
+++ b/src/MaaCore/Vision/FeatureMatcher.h
@@ -0,0 +1,58 @@
+#pragma once
+#include "Vision/Config/FeatureMatcherConfig.h"
+#include "VisionHelper.h"
+
+#include
+#include
+#include
+
+// MAA_SUPPRESS_CV_WARNINGS_BEGIN
+#include
+
+// MAA_SUPPRESS_CV_WARNINGS_END
+
+namespace asst
+{
+class FeatureMatcher : public VisionHelper, public FeatureMatcherConfig
+{
+public:
+ using Result = FeatureMatchRect;
+ using ResultsVec = std::vector;
+ using ResultsVecOpt = std::optional;
+
+public:
+ using VisionHelper::VisionHelper;
+ virtual ~FeatureMatcher() override = default;
+
+ // analyze函数用于分析图像，返回匹配结果
+ ResultsVecOpt analyze() const;
+
+ // FIXME: 老接口太难重构了，先弄个这玩意兼容下，后续慢慢全删掉
+ const auto& get_result() const noexcept { return m_result; }
+
+protected:
+ virtual void _set_roi(const Rect& roi) override { set_roi(roi); }
+
+private:
+ // FIXME: 老接口太难重构了，先弄个这玩意兼容下，后续慢慢全删掉
+ mutable ResultsVec m_result;
+
+private:
+ std::pair, cv::Mat> detect(const cv::Mat& image, const cv::Mat& mask) const;
+ asst::FeatureMatcher::ResultsVec feature_match(
+ const cv::Mat& templ,
+ const std::vector& keypoints_1,
+ const cv::Mat& descriptors_1) const;
+ std::vector> match(const cv::Mat& descriptors_1, const cv::Mat& descriptors_2) const;
+ asst::FeatureMatcher::ResultsVec feature_postproc(
+ const std::vector>& match_points,
+ const std::vector& keypoints_1,
+ const std::vector& keypoints_2,
+ int templ_cols,
+ int templ_rows,
+ std::vector& good_matches) const;
+ cv::Ptr create_detector() const;
+ cv::Ptr create_matcher() const;
+};
+
+} // namespace asst
diff --git a/src/MaaCore/Vision/VisionHelper.cpp b/src/MaaCore/Vision/VisionHelper.cpp
index be38d36eb5..969db75c94 100644
--- a/src/MaaCore/Vision/VisionHelper.cpp
+++ b/src/MaaCore/Vision/VisionHelper.cpp
@@ -80,6 +80,23 @@ Rect VisionHelper::correct_rect(const Rect& rect, const cv::Mat& image)
return res;
}
+cv::Mat asst::VisionHelper::create_mask(const cv::Mat& image, bool green_mask)
+{
+ cv::Mat mask = cv::Mat::ones(image.size(), CV_8UC1);
+ if (green_mask) {
+ cv::inRange(image, cv::Scalar(0, 255, 0), cv::Scalar(0, 255, 0), mask);
+ mask = ~mask;
+ }
+ return mask;
+}
+
+cv::Mat asst::VisionHelper::create_mask(const cv::Mat& image, const cv::Rect& roi)
+{
+ cv::Mat mask = cv::Mat::zeros(image.size(), CV_8UC1);
+ mask(roi) = 255;
+ return mask;
+}
+
bool VisionHelper::save_img(const std::filesystem::path& relative_dir)
{
std::string stem = utils::get_time_filestem();
@@ -93,3 +110,33 @@ bool VisionHelper::save_img(const std::filesystem::path& relative_dir)
return ret;
}
+
+cv::Mat VisionHelper::draw_roi(const cv::Rect& roi, const cv::Mat& base) const
+{
+ cv::Mat image_draw = base.empty() ? m_image.clone() : base;
+ const cv::Scalar color(0, 255, 0);
+
+ // cv::putText(image_draw, name_, cv::Point(5, m_image.rows - 5), cv::FONT_HERSHEY_SIMPLEX, 1, color, 2);
+
+ cv::rectangle(image_draw, roi, color, 1);
+ // std::string flag = MAA_FMT::format("ROI: [{}, {}, {}, {}]", roi.x, roi.y, roi.width, roi.height);
+ std::string flag = "ROI: [" + std::to_string(roi.x) + ", " + std::to_string(roi.y) + ", " +
+ std::to_string(roi.width) + ", " + std::to_string(roi.height) + "]";
+ cv::putText(image_draw, flag, cv::Point(roi.x, roi.y - 5), cv::FONT_HERSHEY_PLAIN, 1.2, color, 1);
+
+ return image_draw;
+}
+
+/*
+void VisionHelper::handle_draw(const cv::Mat& draw) const
+{
+
+ if (show_draw_) {
+ const std::string kWinName = "Draw";
+ cv::imshow(kWinName, draw);
+ cv::waitKey(0);
+ cv::destroyWindow(kWinName);
+ }
+
+}
+*/
diff --git a/src/MaaCore/Vision/VisionHelper.h b/src/MaaCore/Vision/VisionHelper.h
index af74df137a..925592f89b 100644
--- a/src/MaaCore/Vision/VisionHelper.h
+++ b/src/MaaCore/Vision/VisionHelper.h
@@ -6,6 +6,10 @@
#include "Utils/Platform.hpp"
#include "Utils/Ranges.hpp"
+#if __has_include()
+#define MAA_VISION_HAS_XFEATURES2D
+#endif
+
// #ifndef ASST_DEBUG
// #define ASST_DEBUG
// #endif // ! ASST_DEBUG
@@ -41,6 +45,9 @@ protected:
protected:
static Rect correct_rect(const Rect& rect, const cv::Mat& image);
+ static cv::Mat create_mask(const cv::Mat& image, bool green_mask);
+ static cv::Mat create_mask(const cv::Mat& image, const cv::Rect& roi);
+ cv::Mat draw_roi(const cv::Rect& roi, const cv::Mat& base) const;
cv::Mat m_image;
#ifdef ASST_DEBUG