fz-mirrors/MaaAssistantArknights
0
0
Fork 0
mirror of https://github.com/MaaAssistantArknights/MaaAssistantArknights.git synced 2026-07-15 17:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

添加对identityArea字段的支持，仅识别指定区域。同时优化Identity部分接口

Browse Source
This commit is contained in:
MistEO
2021-09-13 01:17:38 +08:00
parent 17bb6b4b07
commit ba91e81eef
11 changed files with 116 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
resource/config.json
View File

@@ -284,6 +284,12 @@
"action": "clickSelf",
"cache": false,
"rearDelay": 3000,
"identifyArea": [
1080,
570,
195,
130
],
"exceededNext": [
"ReturnToMall"
],
@@ -298,7 +304,13 @@
"algorithm": "OcrDetect",
"text": [ "访问下位" ],
"action": "clickSelf",
"rearDelay": 3000,
"rearDelay": 3000,
"identifyArea": [
1080,
570,
195,
130
],
"exceededNext": [
"ReturnToMall"
],
@@ -366,6 +378,12 @@
"algorithm": "OcrDetect",
"text": [ "今日参与", "已达上限" ],
"action": "doNothing",
"identifyArea": [
900,
50,
375,
140
],
"next": [
"ReturnToMall"
]
@@ -373,6 +391,12 @@
"VisitNextBlack": {
"template": "VisitNextBlack.png",
"action": "doNothing",
"identifyArea": [
1080,
570,
195,
130
],
"next": [
"ReturnToMall"
]
@@ -557,6 +581,12 @@
"text": [ "可收获", "订单交付", "信赖" ],
"rearDelay": 1000,
"action": "clickSelf",
"identifyArea": [
0,
600,
800,
118
],
"next": [
"InfrastReward",
"InfrastExitReward"

BIN
resource/operators/Elite1.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 36 KiB

BIN
resource/operators/Elite2.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 36 KiB

6
src/MeoAssistance/AsstAux.h
View File

@@ -79,6 +79,12 @@ namespace asst {
return strTemp;
}
template<typename RetTy, typename ArgType>
constexpr inline RetTy make_rect(const ArgType& rect)
{
return RetTy{ rect.x, rect.y, rect.width, rect.height };
}
//template<typename T,
// typename = typename std::enable_if<std::is_constructible<T, std::string>::value>::type>
// std::string VectorToString(const std::vector<T>& vector, bool to_gbk = false) {

1
src/MeoAssistance/AsstDef.h
View File

@@ -134,6 +134,7 @@ namespace asst {
int pre_delay = 0; // 执行该任务前的延时
int rear_delay = 0; // 执行该任务后的延时
int retry_times = INT_MAX; // 未找到图像时的重试次数
Rect identify_area; // 要识别的区域，若为0则全图识别
};
// 文字识别任务的信息

8
src/MeoAssistance/Configer.cpp
View File

@@ -182,6 +182,14 @@ bool asst::Configer::parse(json::value&& json)
task_info_ptr->reduce_other_times.emplace_back(reduce.as_string());
}
}
if (task_json.exist("identifyArea")) {
json::array& area_arr = task_json["identifyArea"].as_array();
task_info_ptr->identify_area = Rect(
area_arr[0].as_integer(),
area_arr[1].as_integer(),
area_arr[2].as_integer(),
area_arr[3].as_integer());
}
json::array& next_arr = task_json["next"].as_array();
for (const json::value& next : next_arr) {

68
src/MeoAssistance/Identify.cpp
View File

@@ -17,22 +17,22 @@ using namespace cv::xfeatures2d;
bool Identify::add_image(const std::string& name, const std::string& path)
{
Mat mat = imread(path);
if (mat.empty()) {
Mat image = imread(path);
if (image.empty()) {
return false;
}
m_mat_map.emplace(name, mat);
m_mat_map.emplace(name, image);
return true;
}
bool asst::Identify::add_text_image(const std::string& text, const std::string& path)
{
Mat mat = imread(path);
if (mat.empty()) {
Mat image = imread(path);
if (image.empty()) {
return false;
}
m_feature_map.emplace(text, surf_detect(mat));
m_feature_map.emplace(text, surf_detect(image));
return true;
}
@@ -73,21 +73,11 @@ double Identify::image_hist_comp(const cv::Mat& src, const cv::MatND& hist)
return 1 - compareHist(image_2_hist(src), hist, CV_COMP_BHATTACHARYYA);
}
asst::Rect asst::Identify::cvrect_2_rect(const cv::Rect& cvRect)
{
return asst::Rect(cvRect.x, cvRect.y, cvRect.width, cvRect.height);
}
cv::Rect asst::Identify::rect_2_cvrect(const asst::Rect& rect)
{
return cv::Rect(rect.x, rect.y, rect.width, rect.height);
}
std::pair<std::vector<cv::KeyPoint>, cv::Mat> asst::Identify::surf_detect(const cv::Mat& mat)
std::pair<std::vector<cv::KeyPoint>, cv::Mat> asst::Identify::surf_detect(const cv::Mat& image)
{
// 灰度图转换
cv::Mat mat_gray;
cv::cvtColor(mat, mat_gray, cv::COLOR_RGB2GRAY);
cv::cvtColor(image, mat_gray, cv::COLOR_RGB2GRAY);
constexpr int min_hessian = 400;
// SURF特征点检测
@@ -243,9 +233,9 @@ std::optional<asst::Rect> asst::Identify::feature_match(
return std::nullopt;
}
std::vector<TextArea> asst::Identify::ocr_detect(const cv::Mat& mat)
std::vector<TextArea> asst::Identify::ocr_detect(const cv::Mat& image)
{
OcrResult ocr_results = m_ocr_lite.detect(mat,
OcrResult ocr_results = m_ocr_lite.detect(image,
50, 0,
0.2f, 0.3f,
2.0f, false, false);
@@ -294,7 +284,7 @@ asst::Identify::FindImageResult asst::Identify::find_image(
if (m_use_cache && m_cache_map.find(templ_name) != m_cache_map.cend()) {
const auto& [raw_rect, hist] = m_cache_map.at(templ_name);
double value = image_hist_comp(image(raw_rect), hist);
Rect dst_rect = cvrect_2_rect(raw_rect);
Rect dst_rect = make_rect<asst::Rect>(raw_rect);
if (rect_zoom) {
dst_rect = dst_rect.center_zoom(0.8);
}
@@ -307,7 +297,7 @@ asst::Identify::FindImageResult asst::Identify::find_image(
if (m_use_cache && value >= add_cache_thres) {
m_cache_map.emplace(templ_name, std::make_pair(raw_rect, image_2_hist(image(raw_rect))));
}
Rect dst_rect = cvrect_2_rect(raw_rect);
Rect dst_rect = make_rect<asst::Rect>(raw_rect);
if (rect_zoom) {
dst_rect = dst_rect.center_zoom(0.8);
}
@@ -370,7 +360,7 @@ std::vector<asst::Identify::FindImageResult> asst::Identify::find_all_images(
#ifdef LOG_TRACE
cv::Mat draw_mat = image.clone();
for (const auto& info : results) {
cv::rectangle(draw_mat, rect_2_cvrect(info.rect), cv::Scalar(0, 0, 255), 1);
cv::rectangle(draw_mat, make_rect<cv::Rect>(info.rect), cv::Scalar(0, 0, 255), 1);
cv::putText(draw_mat, std::to_string(info.score), cv::Point(info.rect.x, info.rect.y), 1, 1.0, cv::Scalar(0, 0, 255));
}
#endif
@@ -378,26 +368,26 @@ std::vector<asst::Identify::FindImageResult> asst::Identify::find_all_images(
return results;
}
std::optional<TextArea> asst::Identify::feature_match(const cv::Mat& mat, const std::string& key)
std::optional<TextArea> asst::Identify::feature_match(const cv::Mat& image, const std::string& templ_name)
{
//DebugTraceFunction;
if (m_feature_map.find(key) == m_feature_map.cend()) {
if (m_feature_map.find(templ_name) == m_feature_map.cend()) {
return std::nullopt;
}
auto&& [query_keypoints, query_mat_vector] = m_feature_map[key];
auto&& [train_keypoints, train_mat_vector] = surf_detect(mat);
auto&& [query_keypoints, query_mat_vector] = m_feature_map[templ_name];
auto&& [train_keypoints, train_mat_vector] = surf_detect(image);
#ifdef LOG_TRACE
cv::Mat query_mat = cv::imread(GetResourceDir() + "operators\\" + Utf8ToGbk(key) + ".png");
cv::Mat query_mat = cv::imread(GetResourceDir() + "operators\\" + Utf8ToGbk(templ_name) + ".png");
auto&& ret = feature_match(query_keypoints, query_mat_vector, train_keypoints, train_mat_vector,
query_mat, mat);
query_mat, image);
#else
auto&& ret = feature_match(query_keypoints, query_mat_vector, train_keypoints, train_mat_vector);
#endif
if (ret) {
TextArea dst;
dst.text = key;
dst.text = templ_name;
dst.rect = std::move(ret.value());
return dst;
}
@@ -440,9 +430,9 @@ bool asst::Identify::ocr_init_models(const std::string& dir)
return false;
}
std::optional<asst::Rect> asst::Identify::find_text(const cv::Mat& mat, const std::string& text)
std::optional<asst::Rect> asst::Identify::find_text(const cv::Mat& image, const std::string& text)
{
std::vector<TextArea> results = ocr_detect(mat);
std::vector<TextArea> results = ocr_detect(image);
for (const TextArea& res : results) {
if (res.text == text) {
return res.rect;
@@ -451,10 +441,10 @@ std::optional<asst::Rect> asst::Identify::find_text(const cv::Mat& mat, const st
return std::nullopt;
}
std::vector<TextArea> asst::Identify::find_text(const cv::Mat& mat, const std::vector<std::string>& texts)
std::vector<TextArea> asst::Identify::find_text(const cv::Mat& image, const std::vector<std::string>& texts)
{
std::vector<TextArea> dst;
std::vector<TextArea> detect_result = ocr_detect(mat);
std::vector<TextArea> detect_result = ocr_detect(image);
for (TextArea& res : detect_result) {
for (const std::string& t : texts) {
if (res.text == t) {
@@ -465,10 +455,10 @@ std::vector<TextArea> asst::Identify::find_text(const cv::Mat& mat, const std::v
return dst;
}
std::vector<TextArea> asst::Identify::find_text(const cv::Mat& mat, const std::unordered_set<std::string>& texts)
std::vector<TextArea> asst::Identify::find_text(const cv::Mat& image, const std::unordered_set<std::string>& texts)
{
std::vector<TextArea> dst;
std::vector<TextArea> detect_result = ocr_detect(mat);
std::vector<TextArea> detect_result = ocr_detect(image);
for (TextArea& res : detect_result) {
for (const std::string& t : texts) {
if (res.text == t) {
@@ -482,10 +472,10 @@ std::vector<TextArea> asst::Identify::find_text(const cv::Mat& mat, const std::u
/*
std::pair<double, asst::Rect> Identify::findImageWithFile(const cv::Mat& cur, const std::string& filename)
{
Mat mat = imread(filename);
if (mat.empty()) {
Mat image = imread(filename);
if (image.empty()) {
return { 0, asst::Rect() };
}
return findImage(cur, mat);
return findImage(cur, image);
}
*/

21
src/MeoAssistance/Identify.h
View File

@@ -41,27 +41,26 @@ namespace asst {
const cv::Mat& image, const std::string& templ_name, double add_cache_thres = NotAddCache, bool rect_zoom = true);
std::vector<FindImageResult> find_all_images(
const cv::Mat& image, const std::string& templ_name, double threshold = 0, bool rect_zoom = true) const;
// return pair< suitability, raw opencv::point>
std::pair<double, cv::Point> match_template(const cv::Mat& cur, const cv::Mat& templ);
std::optional<TextArea> feature_match(const cv::Mat& mat, const std::string& key);
std::optional<TextArea> feature_match(const cv::Mat& image, const std::string& key);
void clear_cache();
/*** OcrLite package ***/
void set_ocr_param(int gpu_index, int thread_number);
bool ocr_init_models(const std::string& dir);
std::optional<Rect> find_text(const cv::Mat& mat, const std::string& text);
std::vector<TextArea> find_text(const cv::Mat& mat, const std::vector<std::string>& texts);
std::vector<TextArea> find_text(const cv::Mat& mat, const std::unordered_set<std::string>& texts);
std::vector<TextArea> ocr_detect(const cv::Mat& mat);
std::vector<TextArea> ocr_detect(const cv::Mat& image);
[[deprecated]] std::optional<Rect> find_text(const cv::Mat& image, const std::string& text);
[[deprecated]] std::vector<TextArea> find_text(const cv::Mat& image, const std::vector<std::string>& texts);
[[deprecated]] std::vector<TextArea> find_text(const cv::Mat& image, const std::unordered_set<std::string>& texts);
private:
cv::Mat image_2_hist(const cv::Mat& src);
double image_hist_comp(const cv::Mat& src, const cv::MatND& hist);
static asst::Rect cvrect_2_rect(const cv::Rect& cvRect);
static cv::Rect rect_2_cvrect(const asst::Rect& rect);
// return pair< suitability, raw opencv::point>
std::pair<double, cv::Point> match_template(const cv::Mat& image, const cv::Mat& templ);
// return pair<特征点s，特征点描述子（向量）>
std::pair<std::vector<cv::KeyPoint>, cv::Mat> surf_detect(const cv::Mat& mat);

20
src/MeoAssistance/IdentifyOperTask.cpp
View File

@@ -161,24 +161,8 @@ int asst::IdentifyOperTask::detect_elite(const cv::Mat& image, const asst::Rect
elite_rect.height = image.rows * 0.1;
cv::Mat elite_mat = image(elite_rect);
// for debug
// 这两个图是在2560*1440下截的，准备做模板匹配，所以要缩放一下
// TODO：后面再弄完整的工程化，先简单缩放下
static cv::Mat elite1 = cv::imread(GetResourceDir() + "operators\\Elite1.png");
static cv::Mat elite2 = cv::imread(GetResourceDir() + "operators\\Elite2.png");
static bool scaled = false;
if (!scaled) {
scaled = true;
double ratio = image.rows / 1440.0;
cv::Size size1(elite1.size().width * ratio, elite1.size().height * ratio);
cv::resize(elite1, elite1, size1);
cv::Size size2(elite2.size().width * ratio, elite2.size().height * ratio);
cv::resize(elite2, elite2, size2);
}
auto&& [score1, point1] = m_identify_ptr->match_template(elite_mat, elite1);
auto&& [score2, point2] = m_identify_ptr->match_template(elite_mat, elite2);
auto&& [algorithm1, score1, point1] = m_identify_ptr->find_image(elite_mat, "Elite1");
auto&& [algorithm2, score2, point2] = m_identify_ptr->find_image(elite_mat, "Elite2");
if (score1 > score2 && score1 > 0.7) {
return 1;
}

14
src/MeoAssistance/InfrastAbstractTask.cpp
View File

@@ -240,9 +240,9 @@ std::vector<TextArea> asst::InfrastAbstractTask::detect_operators_name(
if (magnified_area.y + magnified_area.height >= image.rows) {
magnified_area.height = image.rows - magnified_area.y - 1;
}
cv::Rect cv_rect(magnified_area.x, magnified_area.y, magnified_area.width, magnified_area.height);
// key是关键字而已，真正要识别的是value
auto&& ret = OcrAbstractTask::m_identify_ptr->feature_match(image(cv_rect), value);
auto&& ret = OcrAbstractTask::m_identify_ptr->feature_match(
image(make_rect<cv::Rect>(magnified_area)), value);
if (ret) {
// 匹配上了下次就不用再匹配这个了，直接删了
all_opers_textarea.emplace_back(value, textarea.rect);
@@ -271,9 +271,8 @@ std::vector<TextArea> asst::InfrastAbstractTask::detect_operators_name(
if (upper_ret) {
TextArea temp = std::move(upper_ret.value());
#ifdef LOG_TRACE // 也顺便涂黑一下，方便看谁没被识别出来
cv::Rect draw_rect(temp.rect.x, temp.rect.y, temp.rect.width, temp.rect.height);
// 注意这里是浅拷贝，原图image也会被涂黑
cv::rectangle(upper_part_name_image, draw_rect, cv::Scalar(0, 0, 0), -1);
cv::rectangle(upper_part_name_image, make_rect<cv::Rect>(temp.rect), cv::Scalar(0, 0, 0), -1);
#endif
// 因为图片是裁剪过的，所以对应原图的坐标要加上裁剪的参数
temp.rect.y += cropped_upper_y;
@@ -287,9 +286,8 @@ std::vector<TextArea> asst::InfrastAbstractTask::detect_operators_name(
if (lower_ret) {
TextArea temp = std::move(lower_ret.value());
#ifdef LOG_TRACE // 也顺便涂黑一下，方便看谁没被识别出来
cv::Rect draw_rect(temp.rect.x, temp.rect.y, temp.rect.width, temp.rect.height);
// 注意这里是浅拷贝，原图image也会被涂黑
cv::rectangle(lower_part_name_image, draw_rect, cv::Scalar(0, 0, 0), -1);
cv::rectangle(lower_part_name_image, make_rect<cv::Rect>(temp.rect), cv::Scalar(0, 0, 0), -1);
#endif
// 因为图片是裁剪过的，所以对应原图的坐标要加上裁剪的参数
temp.rect.y += cropped_lower_y;
@@ -327,7 +325,7 @@ bool asst::InfrastAbstractTask::enter_station(const std::vector<std::string>& te
}
Rect& rect = cur_result.at(0).rect;
callback_json["value"] = cur_result.at(0).score;
callback_json["rect"] = json::array({ rect.x, rect.y, rect.width, rect.height });
callback_json["rect"] = make_rect<json::array>(rect);
m_callback(AsstMsg::ImageFindResult, callback_json, m_callback_arg);
if (max_score_reslut.empty()
@@ -345,7 +343,7 @@ bool asst::InfrastAbstractTask::enter_station(const std::vector<std::string>& te
Rect& rect = max_score_reslut.at(0).rect;
callback_json["size"] = max_score_reslut.size();
callback_json["value"] = max_score_reslut.at(0).score;
callback_json["rect"] = json::array({ rect.x, rect.y, rect.width, rect.height });
callback_json["rect"] = make_rect<json::array>(rect);
m_callback(AsstMsg::ImageMatched, callback_json, m_callback_arg);
if (index >= max_score_reslut.size()) {

27
src/MeoAssistance/ProcessTask.cpp
View File

@@ -157,9 +157,18 @@ std::shared_ptr<TaskInfo> ProcessTask::match_image(Rect* matched_rect)
double templ_threshold = process_task_info_ptr->templ_threshold;
double hist_threshold = process_task_info_ptr->hist_threshold;
double add_cache_thres = process_task_info_ptr->cache ? templ_threshold : Identify::NotAddCache;
auto&& [algorithm, score, temp_rect] = m_identify_ptr->find_image(cur_image, task_name, add_cache_thres);
cv::Mat identify_image;
const auto& identify_area = task_info_ptr->identify_area;
if (identify_area.width == 0) {
identify_image = cur_image;
}
else {
identify_image = cur_image(make_rect<cv::Rect>(task_info_ptr->identify_area));
}
auto&& [algorithm, score, temp_rect] = m_identify_ptr->find_image(identify_image, task_name, add_cache_thres);
rect = std::move(temp_rect);
rect.x += identify_area.x;
rect.y += identify_area.y;
callback_json["value"] = score;
if (algorithm == AlgorithmType::MatchTemplate) {
@@ -185,7 +194,15 @@ std::shared_ptr<TaskInfo> ProcessTask::match_image(Rect* matched_rect)
{
std::shared_ptr<OcrTaskInfo> ocr_task_info_ptr =
std::dynamic_pointer_cast<OcrTaskInfo>(task_info_ptr);
std::vector<TextArea> all_text_area = ocr_detect(cur_image);
cv::Mat identify_image;
const auto& identify_area = task_info_ptr->identify_area;
if (identify_area.width == 0) {
identify_image = cur_image;
}
else {
identify_image = cur_image(make_rect<cv::Rect>(task_info_ptr->identify_area));
}
std::vector<TextArea> all_text_area = ocr_detect(identify_image);
std::vector<TextArea> match_result;
if (ocr_task_info_ptr->need_match) {
match_result = text_match(all_text_area,
@@ -200,6 +217,8 @@ std::shared_ptr<TaskInfo> ProcessTask::match_image(Rect* matched_rect)
if (!match_result.empty()) {
callback_json["text"] = match_result.at(0).text;
rect = match_result.at(0).rect;
rect.x += identify_area.x;
rect.y += identify_area.y;
matched = true;
}
}
@@ -212,7 +231,7 @@ std::shared_ptr<TaskInfo> ProcessTask::match_image(Rect* matched_rect)
break;
}
callback_json["rect"] = json::array({ rect.x, rect.y, rect.width, rect.height });
callback_json["rect"] = make_rect<json::array>(rect);
callback_json["name"] = task_name;
if (matched_rect != NULL) {
*matched_rect = std::move(rect);