封装了识别干员的TASK

MeoAssistance/Assistance.cpp

@@ -243,6 +243,28 @@ bool Assistance::start_open_recruit(const std::vector<int>& required_level, bool
 	return true;
 }
 
+bool asst::Assistance::start_to_identify_opers()
+{
+	DebugTraceFunction;
+	if (!m_thread_idle || !m_inited)
+	{
+		return false;
+	}
+
+	std::unique_lock<std::mutex> lock(m_mutex);
+
+	auto task_ptr = std::make_shared<IdentifyOperTask>(task_callback, (void*)this);
+	// TODO 这个参数写到配置文件里，TODO 滑动位置要根据分辨率缩放
+	task_ptr->set_swipe_param(Rect(2400, 800, 0, 0), Rect(400, 800, 0, 0), 2100);
+	task_ptr->set_task_chain("IdentifyOpers");
+	m_tasks_queue.emplace(task_ptr);
+
+	m_thread_idle = false;
+	m_condvar.notify_one();
+
+	return true;
+}
+
 bool asst::Assistance::start_infrast()
 {
 	DebugTraceFunction;

MeoAssistance/Assistance.h

@@ -34,6 +34,8 @@ namespace asst {
 
 		// 开始公开招募操作
 		bool start_open_recruit(const std::vector<int>& required_level, bool set_time = true);
+		// 开始干员识别任务
+		bool start_to_identify_opers();
 		// 开始基建换班任务
 		bool start_infrast();
 

MeoAssistance/AsstCaller.cpp

@@ -129,14 +129,14 @@ bool AsstTestOcr(asst::Assistance* p_asst, const char** text_array, int array_si
 		return false;
 	}
 
-	std::vector<std::string> text_vec;
-	for (int i = 0; i != array_size; ++i) {
-		if (text_array[i] == nullptr) {
-			return false;
-		}
-		text_vec.emplace_back(asst::GbkToUtf8(text_array[i]));
-	}
-	p_asst->start_ocr_test_task(std::move(text_vec), need_click);
+	//std::vector<std::string> text_vec;
+	//for (int i = 0; i != array_size; ++i) {
+	//	if (text_array[i] == nullptr) {
+	//		return false;
+	//	}
+	//	text_vec.emplace_back(asst::GbkToUtf8(text_array[i]));
+	//}
+	p_asst->start_to_identify_opers();
 	return true;
 }
 

MeoAssistance/IdentifyOperTask.cpp

@@ -0,0 +1,175 @@
+#include "IdentifyOperTask.h"
+
+#include <functional>
+#include <thread>
+#include <future>
+#include <unordered_map>
+#include <unordered_set>
+
+#include <opencv2/opencv.hpp>
+
+#include "Configer.h"
+#include "InfrastConfiger.h"
+#include "Identify.h"
+#include "WinMacro.h"
+
+using namespace asst;
+
+asst::IdentifyOperTask::IdentifyOperTask(AsstCallback callback, void* callback_arg)
+	: OcrAbstractTask(callback, callback_arg)
+{
+	;
+}
+
+bool asst::IdentifyOperTask::run()
+{
+	if (m_view_ptr == nullptr
+		|| m_identify_ptr == nullptr
+		|| m_control_ptr == nullptr)
+	{
+		m_callback(AsstMsg::PtrIsNull, json::value(), m_callback_arg);
+		return false;
+	}
+
+	json::value task_start_json = json::object{
+		{ "task_type",  "InfrastStationTask" },
+		{ "task_chain", OcrAbstractTask::m_task_chain},
+	};
+	m_callback(AsstMsg::TaskStart, task_start_json, m_callback_arg);
+
+	std::unordered_map<std::string, std::string> feature_cond = InfrastConfiger::get_instance().m_oper_name_feat;
+	std::unordered_set<std::string> feature_whatever = InfrastConfiger::get_instance().m_oper_name_feat_whatever;
+	std::unordered_set<OperInfrastInfo> detected_opers;
+	//auto swipe_foo = std::bind(&IdentifyOperTask::swipe, *this);
+
+	// 一边识别一边滑动，把所有干员名字抓出来
+	while (true) {
+		const cv::Mat& image = OcrAbstractTask::get_format_image(true);
+		// 异步进行滑动操作
+		std::future<bool> swipe_future = std::async(std::launch::async, &IdentifyOperTask::swipe, this, false);
+
+		auto cur_name_textarea = detect_opers(image, feature_cond, feature_whatever);
+
+		int oper_numer = detected_opers.size();
+		for (const TextArea& textarea : cur_name_textarea)
+		{
+			cv::Rect elite_rect;
+			// 因为有的名字长有的名字短，但是右对齐的，所以跟着右边走
+			// TODO，这些长宽的参数要跟着分辨率缩放，最好放到配置文件里
+			elite_rect.x = textarea.rect.x + textarea.rect.width - 250;
+			elite_rect.y = textarea.rect.y - 200;
+			if (elite_rect.x < 0 || elite_rect.y < 0) {
+				continue;
+			}
+			elite_rect.width = 100;
+			elite_rect.height = 150;
+			cv::Mat elite_mat = image(elite_rect);
+
+			// for debug
+			static cv::Mat elite1 = cv::imread(GetResourceDir() + "operators\\Elite1.png");
+			static cv::Mat elite2 = cv::imread(GetResourceDir() + "operators\\Elite2.png");
+			auto&& [score1, point1] = OcrAbstractTask::m_identify_ptr->match_template(elite_mat, elite1);
+			auto&& [score2, point2] = OcrAbstractTask::m_identify_ptr->match_template(elite_mat, elite2);
+#ifdef LOG_TRACE
+			std::cout << "elite1:" << score1 << ", elite2:" << score2 << std::endl;
+#endif
+			OperInfrastInfo info;
+			info.name = textarea.text;
+			if (score1 > score2 && score1 > 0.7) {
+				info.elite = 1;
+			}
+			else if (score2 > score1 && score2 > 0.7) {
+				info.elite = 2;
+			}
+			else {
+				info.elite = 0;
+			}
+			detected_opers.emplace(std::move(info));
+		}
+
+		json::value opers_json;
+		std::vector<json::value> opers_json_vec;
+		for (const OperInfrastInfo& info : detected_opers) {
+			json::value info_json;
+			info_json["name"] = Utf8ToGbk(info.name);
+			info_json["elite"] = info.elite;
+			//info_json["level"] = info.level;
+			opers_json_vec.emplace_back(std::move(info_json));
+		}
+		opers_json["all"] = json::array(opers_json_vec);
+		m_callback(AsstMsg::InfrastOpers, opers_json, m_callback_arg);
+
+		// 阻塞等待滑动结束
+		if (!swipe_future.get()) {
+			return false;
+		}
+		// 说明本次识别一个新的都没识别到，应该是滑动到最后了，直接结束循环
+		if (oper_numer == detected_opers.size()) {
+			break;
+		}
+	}
+
+	return true;
+}
+
+std::vector<TextArea> asst::IdentifyOperTask::detect_opers(const cv::Mat& image, std::unordered_map<std::string, std::string>& feature_cond, std::unordered_set<std::string>& feature_whatever)
+{
+	std::vector<TextArea> all_text_area = ocr_detect(image);
+	/* 过滤出所有制造站中的干员名 */
+	std::vector<TextArea> cur_name_textarea = text_search(
+		all_text_area,
+		InfrastConfiger::get_instance().m_all_opers_name,
+		Configer::get_instance().m_infrast_ocr_replace);
+
+	// 用特征检测再筛选一遍OCR识别漏了的
+	for (const TextArea& textarea : all_text_area) {
+		for (auto iter = feature_cond.begin(); iter != feature_cond.end(); ++iter) {
+			auto& [key, value] = *iter;
+			// 识别到了key，但是没识别到value，这种情况就需要进行特征检测进一步确认了
+			if (textarea.text.find(key) != std::string::npos
+				&& textarea.text.find(value) == std::string::npos) {
+				// 把key所在的矩形放大一点送去做特征检测，不需要把整张图片都送去检测
+				Rect magnified_area = textarea.rect.center_zoom(2.0);
+				magnified_area.x = (std::max)(0, magnified_area.x);
+				magnified_area.y = (std::max)(0, magnified_area.y);
+				if (magnified_area.x + magnified_area.width >= image.cols) {
+					magnified_area.width = image.cols - magnified_area.x - 1;
+				}
+				if (magnified_area.y + magnified_area.height >= image.rows) {
+					magnified_area.height = image.rows - magnified_area.y - 1;
+				}
+				cv::Rect cv_rect(magnified_area.x, magnified_area.y, magnified_area.width, magnified_area.height);
+				// key是关键字而已，真正要识别的是value
+				auto&& ret = OcrAbstractTask::m_identify_ptr->feature_match(image(cv_rect), value);
+				if (ret) {
+					cur_name_textarea.emplace_back(value, textarea.rect);
+					iter = feature_cond.erase(iter);
+					--iter;
+				}
+			}
+		}
+	}
+	for (auto iter = feature_whatever.begin(); iter != feature_whatever.end(); ++iter) {
+		auto&& ret = OcrAbstractTask::m_identify_ptr->feature_match(image, *iter);
+		if (ret) {
+			cur_name_textarea.emplace_back(std::move(ret.value()));
+			iter = feature_whatever.erase(iter);
+			--iter;
+		}
+	}
+
+	return cur_name_textarea;
+}
+
+bool IdentifyOperTask::swipe(bool reverse)
+{
+	bool ret = false;
+	if (!reverse) {
+		ret = m_control_ptr->swipe(m_swipe_begin, m_swipe_end, m_swipe_duration);
+	}
+	else {
+		ret = m_control_ptr->swipe(m_swipe_end, m_swipe_begin, m_swipe_duration);
+	}
+	ret &= sleep(m_swipe_duration + SwipeExtraDelay);
+	return ret;
+}

MeoAssistance/IdentifyOperTask.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "OcrAbstractTask.h"
+
+namespace asst {
+	// 识别干员任务，会将识别到的信息写入文件中
+	class IdentifyOperTask : public OcrAbstractTask
+	{
+	public:
+		IdentifyOperTask(AsstCallback callback, void* callback_arg);
+		virtual ~IdentifyOperTask() = default;
+
+		virtual bool run() override;
+		virtual void set_swipe_param(Rect begin, Rect end, int duration) {
+			m_swipe_begin = std::move(begin);
+			m_swipe_end = std::move(end);
+			m_swipe_duration = duration;
+		}
+		void set_filename(const std::string& filename) {
+			m_filename = filename;
+		}
+	protected:
+		constexpr static int SwipeExtraDelay = 100;
+
+		virtual bool swipe(bool reverse = false);
+		std::vector<TextArea> detect_opers(const cv::Mat& image,
+			std::unordered_map<std::string, std::string>& feature_cond,
+			std::unordered_set<std::string>& feature_whatever);
+
+		Rect m_swipe_begin;
+		Rect m_swipe_end;
+		int m_swipe_duration = 0;
+		std::string m_filename;
+	};
+}

MeoAssistance/MeoAssistance.vcxproj

@@ -21,6 +21,7 @@
     <ClInclude Include="ClickTask.h" />
     <ClInclude Include="Configer.h" />
     <ClInclude Include="Identify.h" />
+    <ClInclude Include="IdentifyOperTask.h" />
     <ClInclude Include="InfrastConfiger.h" />
     <ClInclude Include="InfrastStationTask.h" />
     <ClInclude Include="Logger.hpp" />
@@ -28,6 +29,7 @@
     <ClInclude Include="OpenRecruitTask.h" />
     <ClInclude Include="ProcessTask.h" />
     <ClInclude Include="RecruitConfiger.h" />
+    <ClInclude Include="SwipeTask.h" />
     <ClInclude Include="Task.h" />
     <ClInclude Include="TestOcrTask.h" />
     <ClInclude Include="Updater.h" />
@@ -37,6 +39,7 @@
   <ItemGroup>
     <ClCompile Include="AbstractTask.cpp" />
     <ClCompile Include="ClickTask.cpp" />
+    <ClCompile Include="IdentifyOperTask.cpp" />
     <ClCompile Include="InfrastConfiger.cpp" />
     <ClCompile Include="Assistance.cpp" />
     <ClCompile Include="AsstCaller.cpp" />
@@ -47,6 +50,7 @@
     <ClCompile Include="OpenRecruitTask.cpp" />
     <ClCompile Include="ProcessTask.cpp" />
     <ClCompile Include="RecruitConfiger.cpp" />
+    <ClCompile Include="SwipeTask.cpp" />
     <ClCompile Include="TestOcrTask.cpp" />
     <ClCompile Include="Updater.cpp" />
     <ClCompile Include="WinMacro.cpp" />

MeoAssistance/MeoAssistance.vcxproj.filters

@@ -108,6 +108,12 @@
     <ClInclude Include="Updater.h">
       <Filter>头文件\Updater</Filter>
     </ClInclude>
+    <ClInclude Include="IdentifyOperTask.h">
+      <Filter>头文件\Task</Filter>
+    </ClInclude>
+    <ClInclude Include="SwipeTask.h">
+      <Filter>头文件\Task</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="WinMacro.cpp">
@@ -155,6 +161,12 @@
     <ClCompile Include="Updater.cpp">
       <Filter>源文件\Updater</Filter>
     </ClCompile>
+    <ClCompile Include="IdentifyOperTask.cpp">
+      <Filter>源文件\Task</Filter>
+    </ClCompile>
+    <ClCompile Include="SwipeTask.cpp">
+      <Filter>源文件\Task</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\resource\config.json">

MeoAssistance/SwipeTask.cpp

@@ -0,0 +1,29 @@
+#include "SwipeTask.h"
+
+#include "WinMacro.h"
+
+using namespace asst;
+
+SwipeTask::SwipeTask(AsstCallback callback, void* callback_arg)
+	: AbstractTask(callback, callback_arg)
+{
+	m_task_type = TaskType::TaskTypeClick;
+}
+
+bool asst::SwipeTask::run()
+{
+	return swipe();
+}
+
+bool asst::SwipeTask::swipe()
+{
+	bool ret = false;
+	if (!m_reverse) {
+		ret = m_control_ptr->swipe(m_swipe_begin, m_swipe_end, m_swipe_duration);
+	}
+	else {
+		ret = m_control_ptr->swipe(m_swipe_end, m_swipe_begin, m_swipe_duration);
+	}
+	ret &= sleep(m_swipe_duration + SwipeExtraDelay);
+	return ret;
+}

MeoAssistance/SwipeTask.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "AbstractTask.h"
+#include "AsstDef.h"
+
+namespace asst {
+	class SwipeTask : public AbstractTask
+	{
+		SwipeTask(AsstCallback callback, void* callback_arg);
+		virtual ~SwipeTask() = default;
+
+		virtual bool run() override;
+
+		virtual void set_swipe_param(Rect begin, Rect end, int duration) {
+			m_swipe_begin = std::move(begin);
+			m_swipe_end = std::move(end);
+			m_swipe_duration = duration;
+		}
+		virtual void set_reverse(bool reverse) {
+			m_reverse = reverse;
+		}
+	protected:
+		constexpr static int SwipeExtraDelay = 100;
+
+		virtual bool swipe();
+
+		Rect m_swipe_begin;
+		Rect m_swipe_end;
+		int m_swipe_duration = 0;
+		bool m_reverse = false;
+	};
+}

MeoAssistance/Task.h

@@ -4,4 +4,5 @@
 #include "InfrastConfiger.h"
 #include "OpenRecruitTask.h"
 #include "ProcessTask.h"
-#include "TestOcrTask.h"
+#include "TestOcrTask.h"
+#include "IdentifyOperTask.h"

MeoAssistance/WinMacro.cpp

@@ -332,7 +332,7 @@ bool WinMacro::click(const Rect& rect)
 	return click(randPointInRect(rect));
 }
 
-bool asst::WinMacro::swipe(const Point& p1, const Point& p2)
+bool asst::WinMacro::swipe(const Point& p1, const Point& p2, int duration)
 {
 	if (m_handle_type != HandleType::Control || !::IsWindow(m_handle)) {
 		return false;
@@ -349,21 +349,22 @@ bool asst::WinMacro::swipe(const Point& p1, const Point& p2)
 		cur_cmd = StringReplaceAll(cur_cmd, "[y1]", std::to_string(y1));
 		cur_cmd = StringReplaceAll(cur_cmd, "[x2]", std::to_string(x2));
 		cur_cmd = StringReplaceAll(cur_cmd, "[y2]", std::to_string(y2));
+		cur_cmd = StringReplaceAll(cur_cmd, "[duration]", std::to_string(duration));
+
 		return callCmd(cur_cmd, false).has_value();
-		return true;
 	}
 	else {	// TODO
 		return false;
 	}
 }
 
-bool asst::WinMacro::swipe(const Rect& r1, const Rect& r2)
+bool asst::WinMacro::swipe(const Rect& r1, const Rect& r2, int duration)
 {
 	if (m_handle_type != HandleType::Control || !::IsWindow(m_handle)) {
 		return false;
 	}
 
-	return swipe(randPointInRect(r1), randPointInRect(r2));
+	return swipe(randPointInRect(r1), randPointInRect(r2), duration);
 }
 
 void asst::WinMacro::setControlScale(double scale, bool real)

MeoAssistance/WinMacro.h

@@ -26,8 +26,8 @@ namespace asst {
 		bool hideWindow();
 		bool click(const Point& p);
 		bool click(const Rect& rect);
-		bool swipe(const Point& p1, const Point& p2);
-		bool swipe(const Rect& r1, const Rect& r2);
+		bool swipe(const Point& p1, const Point& p2, int duration = 0);
+		bool swipe(const Rect& r1, const Rect& r2, int duration = 0);
 		void setControlScale(double scale, bool real = false);
 		cv::Mat getImage(const Rect& rect);	// 通过Win32 Api对窗口截图
 		cv::Mat getAdbImage();				// 通过Adb截图，会高清一点，但是比较慢（通过adb pull出来，有io操作）

Tools/TestCaller/main.cpp

@@ -22,7 +22,7 @@ int main(int argc, char** argv)
 
 	char ch = 0;
 	while (ch != 'q') {
-		test_infrast(ptr);
+		test_ocr(ptr);
 		//test_swipe(ptr);
 
 		ch = getchar();

resource/config.json

@@ -58,7 +58,7 @@
                 "path": "[EmulatorPath]Engine\\ProgramFiles\\HD-Adb.exe",
                 "connect": "[Adb] connect 127.0.0.1:5555",
                 "click": "[Adb] -s 127.0.0.1:5555 shell input tap [x] [y]",
-                "swipe": "[Adb] -s 127.0.0.1:5555 shell input swipe [x1] [y1] [x2] [y2] 2000",
+                "swipe": "[Adb] -s 127.0.0.1:5555 shell input swipe [x1] [y1] [x2] [y2] [duration]",
                 "display": "[Adb] -s 127.0.0.1:5555 shell dumpsys window displays | grep init= | awk ' { print $3 } '",
                 "displayRegex": "cur=%dx%d",
                 "screencap": "[Adb] -s 127.0.0.1:5555 shell screencap /sdcard/meo_screen.png",
@@ -92,7 +92,7 @@
                 "path": "[EmulatorPath]..\\vmonitor\\bin\\adb_server.exe",
                 "connect": "[Adb] connect 127.0.0.1:7555",
                 "click": "[Adb] -s 127.0.0.1:7555 shell input tap [x] [y]",
-                "swipe": "[Adb] -s 127.0.0.1:7555 shell input swipe [x1] [y1] [x2] [y2] 2000",
+                "swipe": "[Adb] -s 127.0.0.1:7555 shell input swipe [x1] [y1] [x2] [y2] [duration]",
                 "display": "[Adb] -s 127.0.0.1:7555 shell dumpsys window displays | grep init= | awk ' { print $3 } '",
                 "displayRegex": "cur=%dx%d",
                 "screencap": "[Adb] -s 127.0.0.1:7555 shell screencap /sdcard/meo_screen.png",