fz-mirrors/MaaAssistantArknights
0
0
Fork 0
mirror of https://github.com/MaaAssistantArknights/MaaAssistantArknights.git synced 2026-07-16 09:50:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

完成SURF算法的demo

Browse Source
This commit is contained in:
MistEO
2021-08-15 03:35:27 +08:00
parent 839259b13f
commit c9a9fb3656
10 changed files with 128 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
MeoAssistance/include/AsstDef.h
View File

@@ -106,6 +106,7 @@ namespace asst {
std::string swipe;
std::string display;
std::string display_regex;
std::string screencap;
int display_width = 0;
int display_height = 0;
};

2
MeoAssistance/include/Task.h
View File

@@ -139,7 +139,7 @@ namespace asst {
virtual void set_task_chain(std::string name) { m_task_chain = std::move(name); }
virtual const std::string& get_task_chain() { return m_task_chain; }
protected:
virtual cv::Mat get_format_image();
virtual cv::Mat get_format_image(bool hd = false); // 参数hd：高清截图，会慢一点
virtual bool set_control_scale(int cur_width, int cur_height);
virtual bool sleep(unsigned millisecond);
virtual bool print_window(const std::string& dir);

5
MeoAssistance/include/WinMacro.h
View File

@@ -28,7 +28,8 @@ namespace asst {
bool swipe(const Point& p1, const Point& p2);
bool swipe(const Rect& r1, const Rect& r2);
void setControlScale(double scale);
cv::Mat getImage(const Rect& rect);
cv::Mat getImage(const Rect& rect); // 通过Win32 Api对窗口截图
cv::Mat getAdbImage(); // 通过Adb截图，会高清一点，但是比较慢（通过adb pull出来，有io操作）
Rect getWindowRect();
const EmulatorInfo& getEmulatorInfo() const noexcept { return m_emulator_info; }
const HandleType& getHandleType() const noexcept { return m_handle_type; }
@@ -45,6 +46,8 @@ namespace asst {
bool m_is_adb = false;
std::string m_click_cmd; // adb点击命令，不是adb的句柄用不到这个
std::string m_swipe_cmd; // adb滑动命令，不是adb的句柄用不到这个
std::string m_screencap_cmd;// adb截图命令，不是adb的句柄用不到这个
std::string m_adb_screen_filename;
std::minstd_rand m_rand_engine;
int m_width = 0;
int m_height = 0;

1
MeoAssistance/src/Configer.cpp
View File

@@ -256,6 +256,7 @@ bool asst::Configer::_load(const std::string& filename)
emulator_info.adb.swipe = emulator_json["adb"]["swipe"].as_string();
emulator_info.adb.display = emulator_json["adb"]["display"].as_string();
emulator_info.adb.display_regex = emulator_json["adb"]["displayRegex"].as_string();
emulator_info.adb.screencap = emulator_json["adb"]["screencap"].as_string();
}
emulator_info.x_offset = emulator_json.get("xOffset", 0);
emulator_info.y_offset = emulator_json.get("yOffset", 0);

44
MeoAssistance/src/Identify.cpp
View File

@@ -82,7 +82,7 @@ std::pair<std::vector<cv::KeyPoint>, cv::Mat> asst::Identify::surf_detect(const
cv::Mat mat_gray;
cv::cvtColor(mat, mat_gray, cv::COLOR_RGB2GRAY);
constexpr int min_hessian = 1000;
constexpr int min_hessian = 4000;
// SURF特征点检测
cv::Ptr<SURF> detector = SURF::create(min_hessian);
std::vector<KeyPoint> keypoints;
@@ -181,30 +181,41 @@ std::vector<TextArea> asst::Identify::feature_matching(const cv::Mat& mat)
}
float maxdist = max_iter->distance;
// 最小的距离
auto min_iter = std::min_element(matches.cbegin(), matches.cend(),
[](const cv::DMatch& lhs, const cv::DMatch& rhs) ->bool {
return lhs.distance < rhs.distance;
}); // 描述符欧式距离（knn）
if (min_iter == matches.cend()) {
continue;
}
float mindist = min_iter->distance;
std::vector<cv::DMatch> good_matches;
constexpr static const double MatchRatio = 0.6;
std::vector<cv::Point> good_points;
constexpr static const double MatchRatio = 0.4;
for (const cv::DMatch dmatch : matches) {
if (dmatch.distance < maxdist * MatchRatio) {
good_matches.emplace_back(dmatch);
if (dmatch.trainIdx >= 0 && dmatch.trainIdx < income_keypoints.size()) {
good_points.emplace_back(income_keypoints.at(dmatch.trainIdx).pt);
}
}
}
// TODO，把这个阈值写到配置文件里
cv::Rect draw_rect;
constexpr static const int MatchSizeThreshold = 10;
if (good_matches.size() >= MatchSizeThreshold) {
if (good_points.size() >= MatchSizeThreshold) {
TextArea textarea;
textarea.text = key;
// TODO
//textarea.rect = ;
int left = 0, right = 0, top = 0, bottom = 0;
for (const cv::Point& pt : good_points) {
if (pt.x < left || left == 0) {
left = pt.x;
}
if (pt.x > right || right == 0) {
right = pt.x;
}
if (pt.y < top || top == 0) {
top = pt.y;
}
if (pt.y > bottom || bottom == 0) {
bottom = pt.y;
}
}
textarea.rect = { left, top, right - left, bottom - top };
draw_rect = { left, top, right - left, bottom - top };
matched_text_area.emplace_back(std::move(textarea));
}
@@ -212,8 +223,7 @@ std::vector<TextArea> asst::Identify::feature_matching(const cv::Mat& mat)
cv::Mat text_mat = cv::imread(GetResourceDir() + "operators\\森蚺.png");
cv::Mat draw_mat;
cv::drawMatches(text_mat, keypoints, mat, income_keypoints, good_matches, draw_mat);
cv::imshow("output", draw_mat);
cv::waitKey(0);
cv::rectangle(draw_mat, draw_rect, cv::Scalar(255, 0, 0), 10);
}
return matched_text_area;
}

170
MeoAssistance/src/Task.cpp
View File

@@ -42,9 +42,15 @@ void AbstractTask::set_exit_flag(bool* exit_flag)
m_exit_flag = exit_flag;
}
cv::Mat AbstractTask::get_format_image()
cv::Mat AbstractTask::get_format_image(bool hd)
{
const cv::Mat& raw_image = m_view_ptr->getImage(m_view_ptr->getWindowRect());
cv::Mat raw_image;
if (hd) {
raw_image = m_view_ptr->getAdbImage();
}
else {
raw_image = m_view_ptr->getImage(m_view_ptr->getWindowRect());
}
if (raw_image.empty()) {
m_callback(AsstMsg::ImageIsEmpty, json::value(), m_callback_arg);
return raw_image;
@@ -54,23 +60,28 @@ cv::Mat AbstractTask::get_format_image()
return raw_image;
}
// 把模拟器边框的一圈裁剪掉
const EmulatorInfo& window_info = m_view_ptr->getEmulatorInfo();
int x_offset = window_info.x_offset;
int y_offset = window_info.y_offset;
int width = raw_image.cols - x_offset - window_info.right_offset;
int height = raw_image.rows - y_offset - window_info.bottom_offset;
if (hd) {
return raw_image;
}
else {
// 把模拟器边框的一圈裁剪掉
const EmulatorInfo& window_info = m_view_ptr->getEmulatorInfo();
int x_offset = window_info.x_offset;
int y_offset = window_info.y_offset;
int width = raw_image.cols - x_offset - window_info.right_offset;
int height = raw_image.rows - y_offset - window_info.bottom_offset;
cv::Mat cropped(raw_image, cv::Rect(x_offset, y_offset, width, height));
cv::Mat cropped(raw_image, cv::Rect(x_offset, y_offset, width, height));
// 根据图像尺寸，调整控制的缩放
set_control_scale(cropped.cols, cropped.rows);
// 根据图像尺寸，调整控制的缩放
set_control_scale(cropped.cols, cropped.rows);
//// 调整尺寸，与资源中截图的标准尺寸一致
//cv::Mat dst;
//cv::resize(cropped, dst, cv::Size(m_configer.DefaultWindowWidth, m_configer.DefaultWindowHeight));
//// 调整尺寸，与资源中截图的标准尺寸一致
//cv::Mat dst;
//cv::resize(cropped, dst, cv::Size(m_configer.DefaultWindowWidth, m_configer.DefaultWindowHeight));
return cropped;
return cropped;
}
}
bool AbstractTask::set_control_scale(int cur_width, int cur_height)
@@ -667,129 +678,6 @@ bool TestOcrTask::run()
return false;
}
json::value task_start_json = json::object{
{ "task_type", "TestOcrTask" },
{ "task_chain", m_task_chain},
};
m_callback(AsstMsg::TaskStart, task_start_json, m_callback_arg);
auto swipe_foo = [&](bool reverse = false) -> bool {
const Point p1(600, 300);
const Point p2(450, 300);
bool ret = false;
if (!reverse) {
ret = m_control_ptr->swipe(p1, p2);
}
else {
ret = m_control_ptr->swipe(p2, p1);
}
ret &= sleep(3000);
return ret;
};
// 识别到的干员名
std::unordered_set<std::string> all_names;
// 一边识别一边滑动，把所有制造站干员名字抓出来
for (int i = 0; i != 20; ++i) {
const cv::Mat& image = get_format_image();
// 异步进行滑动操作
std::future<bool> swipe_future = std::async(std::launch::async, swipe_foo);
std::vector<TextArea> all_text_area = ocr_detect(image);
/* 过滤出所有制造站中的干员名 */
std::vector<TextArea> cur_names = text_search(
all_text_area,
InfrastConfiger::get_instance().m_mfg_opers,
Configer::get_instance().m_infrast_ocr_replace);
for (TextArea& text_area : cur_names) {
all_names.emplace(std::move(text_area.text));
}
bool break_flag = false;
// 识别到了结束标记，就直接退出循环
if (all_names.find(InfrastConfiger::get_instance().m_mfg_end) != all_names.cend()) {
break_flag = true;
}
// 阻塞等待滑动结束
if (!swipe_future.get()) {
return false;
}
if (break_flag) {
break;
}
}
std::cout << "识别到制造站干员:" << std::endl;
for (const std::string& name : all_names) {
std::cout << Utf8ToGbk(name) << std::endl;
}
// 配置文件中的干员组合，和抓出来的干员名比对，如果组合中的干员都有，那就用这个组合
// Todo 时间复杂度起飞了，需要优化下
std::vector<std::string> optimal_comb;
for (auto&& name_vec : InfrastConfiger::get_instance().m_mfg_combs) {
int count = 0;
for (std::string& name : name_vec) {
if (all_names.find(name) != all_names.cend()) {
++count;
}
else {
break;
}
}
if (count == name_vec.size()) {
optimal_comb = name_vec;
break;
}
}
std::cout << "最优组合:" << std::endl;
for (const std::string& name : optimal_comb) {
std::cout << Utf8ToGbk(name) << std::endl;
}
// 一边滑动一边点击最优解中的干员
for (int i = 0; i != 20; ++i) {
const cv::Mat& image = get_format_image();
std::vector<TextArea> all_text_area = ocr_detect(image);
/* 过滤出所有制造站中的干员名 */
std::vector<TextArea> cur_names = text_search(
all_text_area,
optimal_comb,
Configer::get_instance().m_infrast_ocr_replace);
for (TextArea& text_area : cur_names) {
m_control_ptr->click(text_area.rect);
// 点过了就不会再点了，直接从最优解vector里面删了
auto iter = std::find(optimal_comb.begin(), optimal_comb.end(), text_area.text);
if (iter != optimal_comb.end()) {
optimal_comb.erase(iter);
}
}
if (optimal_comb.empty()) {
break;
}
// 因为滑动和点击是矛盾的，这里没法异步做
if (!swipe_foo(true)) {
return false;
}
}
//// 点击识别到的文字，直接回调扔出去，交给外层做
//if (m_need_click) {
// for (const TextArea& text_area : all_text) {
// json::value task_json;
// task_json["type"] = "ClickTask";
// task_json["rect"] = json::array({ text_area.rect.x, text_area.rect.y, text_area.rect.width, text_area.rect.height });
// task_json["retry_times"] = m_retry_times;
// task_json["task_chain"] = m_task_chain;
// m_callback(AsstMsg::AppendTask, task_json, m_callback_arg);
// }
//}
return true;
}
@@ -808,9 +696,9 @@ bool asst::InfrastStationTask::run()
}
// for debug
const cv::Mat& debug_image = get_format_image();
const cv::Mat& debug_image = get_format_image(true);
m_identify_ptr->feature_matching(debug_image);
std::vector<std::vector<std::string>> all_oper_combs; // 所有的干员组合
std::unordered_set<std::string> all_oper_name; // 所有干员名
std::string oper_end_flag; // 干员名结束标记，识别到这个string就认为识别完成了
@@ -890,7 +778,7 @@ bool asst::InfrastStationTask::run()
// 配置文件中的干员组合，和抓出来的干员名比对，如果组合中的干员都有，那就用这个组合
// Todo 时间复杂度起飞了，需要优化下
std::vector<std::string> optimal_comb;
for (auto&& name_vec :all_oper_combs) {
for (auto&& name_vec : all_oper_combs) {
int count = 0;
for (std::string& name : name_vec) {
if (detected_names.find(name) != detected_names.cend()) {

71
MeoAssistance/src/WinMacro.cpp
View File

@@ -32,6 +32,7 @@ bool WinMacro::captured() const noexcept
bool WinMacro::findHandle()
{
std::vector<HandleInfo> handle_vec;
m_is_adb = m_emulator_info.is_adb;
switch (m_handle_type) {
case HandleType::Window:
m_width = m_emulator_info.width;
@@ -96,25 +97,35 @@ bool WinMacro::findHandle()
size_t pos = adb_dir.find_last_of('\\') + 1;
adb_dir = adb_dir.substr(0, pos);
adb_dir = '"' + StringReplaceAll(m_emulator_info.adb.path, "[EmulatorPath]", adb_dir) + '"';
std::string connect_cmd = adb_dir + m_emulator_info.adb.connect;
if (!callCmd(connect_cmd)) {
DebugTraceError("Connect Adb Error", connect_cmd);
return false;
}
auto&& display_ret = callCmd(adb_dir + m_emulator_info.adb.display);
if (display_ret) {
std::string pipe_str = display_ret.value();
sscanf_s(pipe_str.c_str(), m_emulator_info.adb.display_regex.c_str(),
&m_emulator_info.adb.display_width, &m_emulator_info.adb.display_height);
}
else {
DebugTraceError("Get Display Error");
return false;
}
if (m_handle_type == HandleType::Control)
{
std::string connect_cmd = StringReplaceAll(m_emulator_info.adb.connect, "[Adb]", adb_dir);
if (!callCmd(connect_cmd)) {
DebugTraceError("Connect Adb Error", connect_cmd);
return false;
}
auto&& display_ret = callCmd(StringReplaceAll(m_emulator_info.adb.display, "[Adb]", adb_dir));
if (display_ret) {
std::string pipe_str = display_ret.value();
sscanf_s(pipe_str.c_str(), m_emulator_info.adb.display_regex.c_str(),
&m_emulator_info.adb.display_width, &m_emulator_info.adb.display_height);
}
else {
DebugTraceError("Get Display Error");
return false;
}
m_click_cmd = adb_dir + m_emulator_info.adb.click;
m_swipe_cmd = adb_dir + m_emulator_info.adb.swipe;
m_click_cmd = StringReplaceAll(m_emulator_info.adb.click, "[Adb]", adb_dir);
m_swipe_cmd = StringReplaceAll(m_emulator_info.adb.swipe, "[Adb]", adb_dir);
}
else if (m_handle_type == HandleType::View)
{
m_adb_screen_filename = GetCurrentDir() + "adb_screen.png";
m_screencap_cmd = StringReplaceAll(
StringReplaceAll(m_emulator_info.adb.screencap, "[Adb]", adb_dir),
"[Filename]", m_adb_screen_filename);
}
}
DebugTrace("Handle:", m_handle, "Name:", m_emulator_info.name, "Type:", m_handle_type);
@@ -140,7 +151,7 @@ std::optional<std::string> asst::WinMacro::callCmd(const std::string& cmd, bool
BOOL pipe_ret = FALSE;
if (use_pipe) {
pipe_ret = ::CreatePipe(&pipe_read, &pipe_write, &sa_out_pipe, PipeBuffSize);
DebugTrace("Create Pipe ret", pipe_ret);
DebugTrace("Create Pipe ret", pipe_ret ? "True" : "False");
}
// 准备启动ADB进程
@@ -156,7 +167,7 @@ std::optional<std::string> asst::WinMacro::callCmd(const std::string& cmd, bool
DWORD ret = -1;
if (!::CreateProcessA(NULL, const_cast<LPSTR>(cmd.c_str()), NULL, NULL, TRUE, CREATE_NO_WINDOW, NULL, NULL, &startup_info, &process_info)) {
DebugTraceError("Create process error");
DebugTraceError("Call", cmd, "Create process error");
return std::nullopt;
}
::WaitForSingleObject(process_info.hProcess, 30000);
@@ -167,8 +178,10 @@ std::optional<std::string> asst::WinMacro::callCmd(const std::string& cmd, bool
DWORD std_num = 0;
if (::PeekNamedPipe(pipe_read, NULL, 0, NULL, &read_num, NULL) && read_num > 0) {
char pipe_buffer[PipeBuffSize] = { 0 };
::ReadFile(pipe_read, pipe_buffer, read_num, &std_num, NULL);
pipe_str = std::string(pipe_buffer, std_num);
BOOL read_ret = ::ReadFile(pipe_read, pipe_buffer, read_num, &std_num, NULL);
if (read_ret) {
pipe_str = std::string(pipe_buffer, std_num);
}
}
}
@@ -410,4 +423,18 @@ cv::Mat WinMacro::getImage(const Rect& rect)
*/
return dst_mat;
}
}
cv::Mat asst::WinMacro::getAdbImage()
{
if (m_handle_type != HandleType::View || !::IsWindow(m_handle)) {
return cv::Mat();
}
if (m_is_adb) {
if (callCmd(m_screencap_cmd, false).has_value()) {
return cv::imread(m_adb_screen_filename);
}
}
return cv::Mat();
}

32
resource/config.json
View File

@@ -53,23 +53,16 @@
{
"class": "BS2CHINAUI",
"window": "BlueStacks App Player"
},
{
"class": "BS2CHINAUI",
"window": "HOSTWND"
},
{
"class": "",
"window": "BlueStacks Android PluginAndroid"
}
],
"adb": {
"path": "[EmulatorPath]HD-Adb.exe",
"connect": " connect 127.0.0.1:5555",
"click": " -s 127.0.0.1:5555 shell input tap [x] [y]",
"swipe": " -s 127.0.0.1:5555 shell input swipe [x1] [y1] [x2] [y2]",
"display": " -s 127.0.0.1:5555 shell dumpsys window displays | grep init= | awk ' { print $3 } '",
"displayRegex": "cur=%dx%d"
"path": "[EmulatorPath]\\\\Engine\\\\ProgramFiles\\\\HD-Adb.exe",
"connect": "[Adb] connect 127.0.0.1:5555",
"click": "[Adb] -s 127.0.0.1:5555 shell input tap [x] [y]",
"swipe": "[Adb] -s 127.0.0.1:5555 shell input swipe [x1] [y1] [x2] [y2]",
"display": "[Adb] -s 127.0.0.1:5555 shell dumpsys window displays | grep init= | awk ' { print $3 } '",
"displayRegex": "cur=%dx%d",
"screencap": "[Adb] -s 127.0.0.1:5555 shell screencap /sdcard/meo_screen.png & [Adb] -s 127.0.0.1:5555 pull /sdcard/meo_screen.png [Filename]"
},
"xOffset": 11,
"yOffset": 46,
@@ -97,11 +90,12 @@
],
"adb": {
"path": "[EmulatorPath]..\\\\vmonitor\\\\bin\\\\adb_server.exe",
"connect": " connect 127.0.0.1:7555",
"click": " -s 127.0.0.1:7555 shell input tap [x] [y]",
"swipe": " -s 127.0.0.1:7555 shell input swipe [x1] [y1] [x2] [y2]",
"display": " -s 127.0.0.1:7555 shell dumpsys window displays | grep init= | awk ' { print $3 } '",
"displayRegex": "cur=%dx%d"
"connect": "[Adb] connect 127.0.0.1:7555",
"click": "[Adb] -s 127.0.0.1:7555 shell input tap [x] [y]",
"swipe": "[Adb] -s 127.0.0.1:7555 shell input swipe [x1] [y1] [x2] [y2]",
"display": "[Adb] -s 127.0.0.1:7555 shell dumpsys window displays | grep init= | awk ' { print $3 } '",
"displayRegex": "cur=%dx%d",
"screencap": "[Adb] -s 127.0.0.1:5555 shell screencap /sdcard/[ImageFilename] & [Adb] -s 127.0.0.1:5555 pull /sdcard/[ImageFilename] ."
},
"xOffset": 0,
"yOffset": 36,

4
resource/operators/screen.bat
View File

@@ -1,3 +1,3 @@
adb connect 127.0.0.1:5555
adb -s 127.0.0.1:5555 shell screencap /sdcard/screen.png
adb -s 127.0.0.1:5555 pull /sdcard/screen.png .
adb -s 127.0.0.1:5555 shell screencap /sdcard/meo_screen.png
adb -s 127.0.0.1:5555 pull /sdcard/meo_screen.png .

5
update_resource.bat
View File

@@ -1,5 +1,2 @@
xcopy /e /y /i resource x64\RelWithDebInfo\resource
xcopy /e /y /i resource x64\Release\resource
xcopy /e /y /i 3rdPart\resource x64\RelWithDebInfo\resource
xcopy /e /y /i 3rdPart\resource x64\Release\resource
xcopy /e /y /i 3rdPart\resource x64\RelWithDebInfo\resource