diff --git a/resource/onnx/deploy_direction_cls.onnx b/resource/onnx/deploy_direction_cls.onnx new file mode 100644 index 0000000000..659a118dde Binary files /dev/null and b/resource/onnx/deploy_direction_cls.onnx differ diff --git a/resource/onnx/operators_det.onnx b/resource/onnx/operators_det.onnx new file mode 100644 index 0000000000..41aeb8adfa Binary files /dev/null and b/resource/onnx/operators_det.onnx differ diff --git a/resource/onnx/skill_ready_cls.onnx b/resource/onnx/skill_ready_cls.onnx new file mode 100644 index 0000000000..3930a0c82e Binary files /dev/null and b/resource/onnx/skill_ready_cls.onnx differ diff --git a/resource/onnx/skill_ready_rec.onnx b/resource/onnx/skill_ready_rec.onnx deleted file mode 100644 index 873f57f854..0000000000 Binary files a/resource/onnx/skill_ready_rec.onnx and /dev/null differ diff --git a/resource/tasks.json b/resource/tasks.json index 7c6720f900..374365ca3c 100644 --- a/resource/tasks.json +++ b/resource/tasks.json @@ -6199,7 +6199,7 @@ 100, 255 ], - "templThreshold": 0.88 + "templThreshold": 0.7 }, "BattleSpeedUp": { "algorithm": "JustReturn", @@ -6237,6 +6237,11 @@ "template": "empty.png", "templThreshold": 0.8 }, + "BattleAvatarDataForVideo": { + "baseTask": "BattleAvatarData", + "template": "empty.png", + "templThreshold": 0.6 + }, "BattleDroneAvatarData": { "template": "empty.png", "templThreshold": 0.7 @@ -6249,6 +6254,56 @@ 50 ] }, + "BattleAvatarDataForFormation": { + "template": "empty.png", + "templThreshold": 0.6, + "rectMove": [ + 15, + 15, + 30, + 30 + ] + }, + "BattleOperDetailPageFlag": { + "template": "empty.png", + "roi": [ + 0, + 322, + 110, + 16 + ], + "specialParams": [ + 100 + ] + }, + "BattleOperDetailPageOldFlag": { + "baseTask": "BattleOperDetailPageFlag", + "template": "empty.png", + "roi": [ + 0, + 385, + 110, + 14 + ] + }, + "BattleOperBoxRectMove": { + "template": "empty.png", + "rectMove": [ + 0, + -50, + 60, + 60 + ] + }, + "BattleDeployDirectionRectMove": { + "template": "empty.png", + "rectMove": [ + -48, + -48, + 96, + 96 + ] + }, "BattleKillsFlag": { "roi": [ 150, @@ -6348,7 +6403,7 @@ ] }, "BattleOpersFlag": { - "templThreshold": 0.85, + "templThreshold": 0.8, "roi": [ 0, 588, @@ -9907,25 +9962,58 @@ "baseTask": "BattleStartPre", "action": "DoNothing" }, + "BattleFormationOCRNameFlag": { + "templThreshold": 0.6, + "roi": [ + 88, + 264, + 873, + 354 + ] + }, "BattleFormationOperNames": { "baseTask": "CharsNameOcrReplace", "text": [], "roi": [ - 86, - 321, - 945, - 325 + -18, + 40, + 122, + 18 + ] + }, + "BattleFormationOperNamesOldVersion": { + "baseTask": "BattleFormationOperNames", + "text": [], + "roi": [ + 30, + 23, + 75, + 15 ] }, "BattleFormationOperAvatarMove": { "template": "empty.png", "rectMove": [ - -125, + -120, -250, - 125, + 150, 165 ] }, + "BattleHasStarted": { + "template": "empty.png", + "roi": [ + 1178, + 33, + 49, + 39 + ], + "specialParams_Doc": "亮度阈值；点的数量", + "specialParams": [ + 245, + 200 + ] + }, "Reclamation@PopupConfirm": { "Doc": "base_task", "template": "PopupConfirm.png", diff --git a/resource/template/BattleFormationOCRNameFlag.png b/resource/template/BattleFormationOCRNameFlag.png new file mode 100644 index 0000000000..3baa29f9d2 Binary files /dev/null and b/resource/template/BattleFormationOCRNameFlag.png differ diff --git a/src/MaaCore/Assistant.cpp b/src/MaaCore/Assistant.cpp index bb92b2ebf6..a3eeba3912 100644 --- a/src/MaaCore/Assistant.cpp +++ b/src/MaaCore/Assistant.cpp @@ -22,6 +22,7 @@ #include "Task/Interface/SSSCopilotTask.h" #include "Task/Interface/SingleStepTask.h" #include "Task/Interface/StartUpTask.h" +#include "Task/Interface/VideoRecognitionTask.h" #include "Utils/Logger.hpp" #ifdef ASST_DEBUG #include "Task/Interface/DebugTask.h" @@ -165,6 +166,7 @@ asst::Assistant::TaskId asst::Assistant::append_task(const std::string& type, co ASST_ASSISTANT_APPEND_TASK_FROM_STRING_IF_BRANCH(CopilotTask) ASST_ASSISTANT_APPEND_TASK_FROM_STRING_IF_BRANCH(SSSCopilotTask) ASST_ASSISTANT_APPEND_TASK_FROM_STRING_IF_BRANCH(SingleStepTask) + ASST_ASSISTANT_APPEND_TASK_FROM_STRING_IF_BRANCH(VideoRecognitionTask) ASST_ASSISTANT_APPEND_TASK_FROM_STRING_IF_BRANCH(DepotTask) ASST_ASSISTANT_APPEND_TASK_FROM_STRING_IF_BRANCH(ReclamationTask) ASST_ASSISTANT_APPEND_TASK_FROM_STRING_IF_BRANCH(CustomTask) @@ -289,7 +291,7 @@ bool asst::Assistant::start(bool block) LogTraceFunction; Log.trace("Start |", block ? "block" : "non block"); - if (!m_thread_idle || !inited()) { + if (!m_thread_idle) { return false; } std::unique_lock lock; diff --git a/src/MaaCore/Common/AsstBattleDef.h b/src/MaaCore/Common/AsstBattleDef.h index 64b5f5bcd9..0955915236 100644 --- a/src/MaaCore/Common/AsstBattleDef.h +++ b/src/MaaCore/Common/AsstBattleDef.h @@ -145,6 +145,7 @@ namespace asst::battle }; struct OperProps { + std::string id; std::string name; Role role = Role::Unknown; std::array ranges; diff --git a/src/MaaCore/Common/AsstTypes.h b/src/MaaCore/Common/AsstTypes.h index ce1841bc9a..4afbccbcd0 100644 --- a/src/MaaCore/Common/AsstTypes.h +++ b/src/MaaCore/Common/AsstTypes.h @@ -269,6 +269,20 @@ namespace std namespace asst { + template + struct ContainerHasher + { + std::size_t operator()(const CType& container) const + { + using value_type = typename CType::value_type; + std::size_t seed = container.size(); + for (const value_type& e : container) { + seed ^= std::hash()(e) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + } + return seed; + } + }; + enum class AlgorithmType { Invalid = -1, diff --git a/src/MaaCore/Config/Miscellaneous/BattleDataConfig.cpp b/src/MaaCore/Config/Miscellaneous/BattleDataConfig.cpp index 6f2a37e197..acbf834bdb 100644 --- a/src/MaaCore/Config/Miscellaneous/BattleDataConfig.cpp +++ b/src/MaaCore/Config/Miscellaneous/BattleDataConfig.cpp @@ -7,8 +7,9 @@ bool asst::BattleDataConfig::parse(const json::value& json) { - for (const auto& char_data_json : json.at("chars").as_object() | views::values) { + for (const auto& [id, char_data_json] : json.at("chars").as_object()) { battle::OperProps data; + data.id = id; std::string name = char_data_json.at("name").as_string(); data.name = name; static const std::unordered_map RoleMap = { diff --git a/src/MaaCore/Config/Miscellaneous/BattleDataConfig.h b/src/MaaCore/Config/Miscellaneous/BattleDataConfig.h index 30290e6f3c..418f77a312 100644 --- a/src/MaaCore/Config/Miscellaneous/BattleDataConfig.h +++ b/src/MaaCore/Config/Miscellaneous/BattleDataConfig.h @@ -13,6 +13,17 @@ namespace asst public: virtual ~BattleDataConfig() override = default; + static inline const std::string& EmptyId = ""; + + const std::string get_id(const std::string& name) const + { + auto iter = m_chars.find(name); + if (iter == m_chars.cend()) { + return EmptyId; + } + return iter->second.id; + } + battle::Role get_role(const std::string& name) const { auto iter = m_chars.find(name); diff --git a/src/MaaCore/Config/OnnxSession.cpp b/src/MaaCore/Config/OnnxSessions.cpp similarity index 79% rename from src/MaaCore/Config/OnnxSession.cpp rename to src/MaaCore/Config/OnnxSessions.cpp index cbbdc721e6..110264120c 100644 --- a/src/MaaCore/Config/OnnxSession.cpp +++ b/src/MaaCore/Config/OnnxSessions.cpp @@ -1,4 +1,4 @@ -#include "OnnxSession.h" +#include "OnnxSessions.h" #include #include @@ -6,7 +6,7 @@ #include "Utils/Logger.hpp" -bool asst::OnnxSession::load(const std::filesystem::path& path) +bool asst::OnnxSessions::load(const std::filesystem::path& path) { LogTraceFunction; Log.info("load", path); diff --git a/src/MaaCore/Config/OnnxSession.h b/src/MaaCore/Config/OnnxSessions.h similarity index 76% rename from src/MaaCore/Config/OnnxSession.h rename to src/MaaCore/Config/OnnxSessions.h index 062f80fcdc..a7ccc2be22 100644 --- a/src/MaaCore/Config/OnnxSession.h +++ b/src/MaaCore/Config/OnnxSessions.h @@ -8,10 +8,10 @@ namespace asst { - class OnnxSession final : public SingletonHolder, public AbstractResource + class OnnxSessions final : public SingletonHolder, public AbstractResource { public: - virtual ~OnnxSession() override = default; + virtual ~OnnxSessions() override = default; virtual bool load(const std::filesystem::path& path) override; Ort::Session& get(const std::string& key) { return m_sessions.at(key); } diff --git a/src/MaaCore/Config/ResourceLoader.cpp b/src/MaaCore/Config/ResourceLoader.cpp index 803c9e7cc7..7e40a6cbeb 100644 --- a/src/MaaCore/Config/ResourceLoader.cpp +++ b/src/MaaCore/Config/ResourceLoader.cpp @@ -14,7 +14,7 @@ #include "Miscellaneous/RecruitConfig.h" #include "Miscellaneous/StageDropsConfig.h" #include "Miscellaneous/TilePack.h" -#include "OnnxSession.h" +#include "OnnxSessions.h" #include "Roguelike/RoguelikeCopilotConfig.h" #include "Roguelike/RoguelikeRecruitConfig.h" #include "Roguelike/RoguelikeShoppingConfig.h" @@ -67,7 +67,12 @@ bool asst::ResourceLoader::load(const std::filesystem::path& path) LoadResourceAndCheckRet(RoguelikeShoppingConfig, "roguelike"_p / "shopping.json"_p); LoadResourceAndCheckRet(BattleDataConfig, "battle_data.json"_p); LoadResourceAndCheckRet(OcrConfig, "ocr_config.json"_p); - LoadResourceAndCheckRet(OnnxSession, "onnx"_p / "skill_ready_rec.onnx"_p); + // 战斗中技能识别，二分类模型 + LoadResourceAndCheckRet(OnnxSessions, "onnx"_p / "skill_ready_cls.onnx"_p); + // 战斗中部署方向识别，四分类模型 + LoadResourceAndCheckRet(OnnxSessions, "onnx"_p / "deploy_direction_cls.onnx"_p); + // 战斗中干员（血条）检测，yolov8 检测模型 + LoadResourceAndCheckRet(OnnxSessions, "onnx"_p / "operators_det.onnx"_p); /* load resource with json and template files*/ LoadResourceWithTemplAndCheckRet(TaskData, "tasks.json"_p, "template"_p); diff --git a/src/MaaCore/MaaCore.vcxproj b/src/MaaCore/MaaCore.vcxproj index 64f25745f3..96c4cdbd5f 100644 --- a/src/MaaCore/MaaCore.vcxproj +++ b/src/MaaCore/MaaCore.vcxproj @@ -39,7 +39,7 @@ - + @@ -55,7 +55,10 @@ + + + @@ -63,12 +66,17 @@ - + + + + + + @@ -77,8 +85,6 @@ - - @@ -88,6 +94,7 @@ + @@ -186,7 +193,7 @@ - + @@ -197,6 +204,9 @@ + + + @@ -204,11 +214,15 @@ - + + + + + @@ -217,8 +231,6 @@ - - @@ -228,6 +240,7 @@ + diff --git a/src/MaaCore/MaaCore.vcxproj.filters b/src/MaaCore/MaaCore.vcxproj.filters index 18ab9a600a..dee3b29f77 100644 --- a/src/MaaCore/MaaCore.vcxproj.filters +++ b/src/MaaCore/MaaCore.vcxproj.filters @@ -65,9 +65,6 @@ {a5775bd2-7a8a-4bcb-9465-75d899ea8ca1} - - {987541fc-58f0-40eb-8b93-c0cb38881d92} - {97f7690d-5924-405f-a332-831e7dad3f1e} @@ -89,6 +86,12 @@ {64367a8b-0654-47bb-92cc-d451799457a4} + + {28d29141-dd91-4954-96e9-a677be9b2471} + + + {987541fc-58f0-40eb-8b93-c0cb38881d92} + @@ -171,9 +174,6 @@ 源文件\Vision\Infrast - - 源文件\Vision\Miscellaneous - 源文件\Vision\Miscellaneous @@ -465,9 +465,6 @@ 源文件\Vision\Miscellaneous - - 源文件\Vision\Miscellaneous - 源文件\Task @@ -495,9 +492,6 @@ 源文件\Vision - - 源文件\Task\SingleStep - 源文件\Task\Interface @@ -561,12 +555,42 @@ 源文件\Resource\Miscellaneous - - 源文件\Resource - 源文件\Task\Interface + + 源文件\Task\Interface + + + 源文件\Vision\Battle + + + 源文件\Vision\Battle + + + 源文件\Vision\Battle + + + 源文件\Vision\Battle + + + 源文件\Vision\Battle + + + 源文件\Resource\Miscellaneous + + + 源文件\Utils + + + 源文件\Vision + + + 源文件\Task\Experiment + + + 源文件\Task\Experiment + @@ -605,9 +629,6 @@ 源文件\Vision\Infrast - - 源文件\Vision\Miscellaneous - 源文件\Vision\Miscellaneous @@ -848,9 +869,6 @@ 源文件\Vision\Miscellaneous - - 源文件\Vision\Miscellaneous - 源文件\Task @@ -875,9 +893,6 @@ 源文件\Vision - - 源文件\Task\SingleStep - 源文件\Task\Interface @@ -923,11 +938,38 @@ 源文件\Resource\Miscellaneous - - 源文件\Resource - 源文件\Task\Interface + + 源文件\Task\Interface + + + 源文件\Vision\Battle + + + 源文件\Vision\Battle + + + 源文件\Vision\Battle + + + 源文件\Vision\Battle + + + 源文件\Vision\Battle + + + 源文件\Resource\Miscellaneous + + + 源文件\Vision + + + 源文件\Task\Experiment + + + 源文件\Task\Experiment + \ No newline at end of file diff --git a/src/MaaCore/Task/AbstractTask.cpp b/src/MaaCore/Task/AbstractTask.cpp index e76ba26e98..1f8409e37c 100644 --- a/src/MaaCore/Task/AbstractTask.cpp +++ b/src/MaaCore/Task/AbstractTask.cpp @@ -157,8 +157,8 @@ bool asst::AbstractTask::save_img(const std::filesystem::path& relative_dir) if (image.empty()) { return false; } - std::string stem = utils::get_random_filestem(); + std::string stem = utils::get_time_filestem(); auto relative_path = relative_dir / (stem + "_raw.png"); Log.trace("Save image", relative_path); return asst::imwrite(relative_path, image); -} +} \ No newline at end of file diff --git a/src/MaaCore/Task/BattleHelper.cpp b/src/MaaCore/Task/BattleHelper.cpp index f24f637344..a5017804cb 100644 --- a/src/MaaCore/Task/BattleHelper.cpp +++ b/src/MaaCore/Task/BattleHelper.cpp @@ -11,10 +11,10 @@ #include "Utils/ImageIo.hpp" #include "Utils/Logger.hpp" #include "Utils/NoWarningCV.h" +#include "Vision/Battle/BattleImageAnalyzer.h" +#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" #include "Vision/BestMatchImageAnalyzer.h" #include "Vision/MatchImageAnalyzer.h" -#include "Vision/Miscellaneous/BattleImageAnalyzer.h" -#include "Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h" #include "Vision/OcrWithPreprocessImageAnalyzer.h" using namespace asst::battle; @@ -134,6 +134,7 @@ bool asst::BattleHelper::update_deployment(bool init, const cv::Mat& reusable) static const auto cooling_mask_range = Task.get("BattleAvatarCoolingData")->mask_range; avatar_analyzer.set_threshold(cooling_threshold); avatar_analyzer.set_mask_range(cooling_mask_range, true); + avatar_analyzer.set_mask_with_close(true); } else { static const double threshold = Task.get("BattleAvatarData")->templ_threshold; @@ -146,7 +147,7 @@ bool asst::BattleHelper::update_deployment(bool init, const cv::Mat& reusable) avatar_analyzer.append_templ(name, avatar); } if (avatar_analyzer.analyze()) { - set_oper_name(oper, avatar_analyzer.get_result_name()); + set_oper_name(oper, avatar_analyzer.get_result().name); m_cur_deployment_opers.insert_or_assign(oper.name, oper); remove_cooling_from_battlefield(oper); } @@ -403,6 +404,8 @@ bool asst::BattleHelper::check_pause_button(const cv::Mat& reusable) MatchImageAnalyzer battle_flag_analyzer(image); battle_flag_analyzer.set_task_info("BattleOfficiallyBegin"); bool ret = battle_flag_analyzer.analyze(); + BattleImageAnalyzer battle_flag_analyzer_2(image); + ret &= battle_flag_analyzer_2.analyze() && battle_flag_analyzer_2.get_pause_button(); return ret; } diff --git a/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp b/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp new file mode 100644 index 0000000000..76b6afd584 --- /dev/null +++ b/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.cpp @@ -0,0 +1,842 @@ +#include "CombatRecordRecognitionTask.h" + +#include "Config/Miscellaneous/BattleDataConfig.h" +#include "Config/Miscellaneous/TilePack.h" +#include "Config/TaskData.h" +#include "Utils/ImageIo.hpp" +#include "Utils/Logger.hpp" +#include "Utils/NoWarningCV.h" +#include "Utils/Ranges.hpp" +#include "Vision/Battle/BattleDeployDirectionImageAnalyzer.h" +#include "Vision/Battle/BattleFormationImageAnalyzer.h" +#include "Vision/Battle/BattleImageAnalyzer.h" +#include "Vision/Battle/BattleOperatorsImageAnalyzer.h" +#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" +#include "Vision/BestMatchImageAnalyzer.h" +#include "Vision/OcrWithPreprocessImageAnalyzer.h" + +#include +#include + +bool asst::CombatRecordRecognitionTask::set_video_path(const std::filesystem::path& path) +{ + if (!std::filesystem::exists(path)) { + Log.error(__FUNCTION__, "filename not exists", path); + return false; + } + m_video_path = path; + return true; +} + +bool asst::CombatRecordRecognitionTask::set_stage_name(const std::string& stage_name) +{ + m_stage_name = stage_name; + return true; +} + +bool asst::CombatRecordRecognitionTask::_run() +{ + LogTraceFunction; + + auto release_video = [](cv::VideoCapture* video) { + if (video && video->isOpened()) { + video->release(); + } + }; + auto u8_path = utils::path_to_utf8_string(m_video_path); + m_video_ptr = std::shared_ptr(new cv::VideoCapture(u8_path), release_video); + + if (!m_video_ptr->isOpened()) { + Log.error(__FUNCTION__, "video_io open failed", m_video_path); + return false; + } + m_video_fps = m_video_ptr->get(cv::CAP_PROP_FPS); + m_video_frame_count = static_cast(m_video_ptr->get(cv::CAP_PROP_FRAME_COUNT)); + m_battle_start_frame = 0; + m_scale = WindowHeightDefault / m_video_ptr->get(cv::CAP_PROP_FRAME_HEIGHT); + +#ifdef ASST_DEBUG + cv::namedWindow(DrawWindow, cv::WINDOW_AUTOSIZE); +#endif // ASST_DEBUG + + if (!analyze_formation()) { + Log.error(__FUNCTION__, "failed to analyze formation"); + return false; + } + + if (!analyze_stage()) { + Log.error(__FUNCTION__, "unknown stage"); + return false; + } + + if (!analyze_deployment()) { + Log.error(__FUNCTION__, "failed to match deployment"); + return false; + } + + if (!slice_video()) { + Log.error(__FUNCTION__, "failed to slice"); + return false; + } + + ClipInfo* pre_clip_ptr = nullptr; + for (auto& clip : m_clips) { + if (!analyze_clip(clip, pre_clip_ptr)) { + Log.error(__FUNCTION__, "failed to analyze clip"); + return false; + } + if (clip.deployment_changed) { + pre_clip_ptr = &clip; + } + } + + Log.info("full copilot json", m_copilot_json.to_string()); + + std::string filename = "MaaAI_" + m_stage_name + "_" + utils::path_to_utf8_string(m_video_path.stem()) + "_" + + utils::get_time_filestem() + ".json"; + auto filepath = UserDir.get() / "cache" / "CombatRecord" / utils::path(filename); + std::filesystem::create_directories(filepath.parent_path()); + std::ofstream osf(filepath); + osf << m_copilot_json.format(); + osf.close(); + + auto cb_json = basic_info_with_what("Finished"); + cb_json["details"]["filename"] = utils::path_to_utf8_string(filepath); + callback(AsstMsg::SubTaskExtraInfo, cb_json); + +#ifdef ASST_DEBUG + cv::destroyWindow(DrawWindow); +#endif // ASST_DEBUG + + return true; +} + +bool asst::CombatRecordRecognitionTask::analyze_formation() +{ + LogTraceFunction; + callback(AsstMsg::SubTaskStart, basic_info_with_what("OcrFormation")); + + const int skip_count = m_video_fps > m_formation_fps ? static_cast(m_video_fps / m_formation_fps) - 1 : 0; + + BattleFormationImageAnalyzer formation_ananlyzer; + int no_changes_count = 0; + for (size_t i = 0; i < m_video_frame_count; i += skip_frames(skip_count) + 1) { + cv::Mat frame; + *m_video_ptr >> frame; + if (frame.empty()) { + Log.error(i, "frame is empty"); + callback(AsstMsg::SubTaskError, basic_info_with_what("OcrFormation")); + return false; + } + + cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); + + formation_ananlyzer.set_image(frame); + bool analyzed = formation_ananlyzer.analyze(); + show_img(formation_ananlyzer); + // 有些视频会有个过渡或者动画啥的，只取一帧识别的可能不全。多识别几帧 + if (analyzed) { + const auto& cur = formation_ananlyzer.get_result(); + if (cur.size() > m_formation.size()) { + m_formation = cur; + } + else if (++no_changes_count > 5) { + m_formation_end_frame = i; + break; + } + } + else if (!m_formation.empty()) { + m_formation_end_frame = i; + break; + } + } + + Log.info("Formation:", m_formation | views::keys); + auto cb_info = basic_info_with_what("OcrFormation"); + auto& cb_formation = cb_info["details"]["formation"]; + for (const auto& [name, avatar] : m_formation) { + std::vector opers; + opers.emplace_back(battle::OperUsage { name, 0, battle::SkillUsage::NotUse }); + json::object oper_json { { "name", name }, { "skill", 0 }, { "skill_usage", 0 } }; + m_copilot_json["opers"].array_emplace(std::move(oper_json)); + + cb_formation.array_emplace(name); + asst::imwrite(utils::path("debug/video_export/formation/") / utils::path(name + ".png"), avatar); + } + callback(AsstMsg::SubTaskCompleted, cb_info); + + return true; +} + +bool asst::CombatRecordRecognitionTask::analyze_stage() +{ + LogTraceFunction; + + if (m_stage_name.empty()) { + callback(AsstMsg::SubTaskStart, basic_info_with_what("OcrStage")); + + const auto stage_name_task_ptr = Task.get("BattleStageName"); + const int skip_count = m_video_fps > m_stage_ocr_fps ? static_cast(m_video_fps / m_stage_ocr_fps) - 1 : 0; + + for (size_t i = m_formation_end_frame; i < m_video_frame_count; i += skip_frames(skip_count) + 1) { + cv::Mat frame; + *m_video_ptr >> frame; + if (frame.empty()) { + Log.error(i, "frame is empty"); + callback(AsstMsg::SubTaskError, basic_info_with_what("OcrStage")); + return false; + } + + cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); + + OcrWithPreprocessImageAnalyzer stage_analyzer(frame); + stage_analyzer.set_task_info(stage_name_task_ptr); + bool analyzed = stage_analyzer.analyze(); + show_img(stage_analyzer); + + if (!analyzed) { + // BattleImageAnalyzer battle_analyzer(frame); + // if (battle_analyzer.analyze()) { + // Log.error(i, "already start button, but still failed to analyze stage name"); + // m_stage_ocr_end_frame = i; + // callback(AsstMsg::SubTaskError, basic_info_with_what("OcrStage")); + // return false; + // } + continue; + } + stage_analyzer.sort_result_by_score(); + const std::string& text = stage_analyzer.get_result().front().text; + + if (text.empty() || !Tile.contains(text)) { + continue; + } + + m_stage_name = text; + m_stage_ocr_end_frame = i; + break; + } + } + + Log.info("Stage", m_stage_name); + if (m_stage_name.empty() || !Tile.contains(m_stage_name)) { + callback(AsstMsg::SubTaskError, basic_info_with_what("OcrStage")); + return false; + } + m_normal_tile_info = Tile.calc(m_stage_name, false); + + m_copilot_json["stage_name"] = m_stage_name; + m_copilot_json["minimum_required"] = "v4.0.0"; + m_copilot_json["doc"]["title"] = "MAA AI - " + m_stage_name; + m_copilot_json["doc"]["details"] = + "Built at: " + utils::get_format_time() + "\n" + utils::path_to_utf8_string(m_video_path); + + callback(AsstMsg::SubTaskCompleted, basic_info_with_what("OcrStage")); + return true; +} + +bool asst::CombatRecordRecognitionTask::analyze_deployment() +{ + LogTraceFunction; + callback(AsstMsg::SubTaskStart, basic_info_with_what("MatchDeployment")); + + const int skip_count = m_video_fps > m_deployment_fps ? static_cast(m_video_fps / m_deployment_fps) - 1 : 0; + + BattleImageAnalyzer oper_analyzer; + oper_analyzer.set_target(BattleImageAnalyzer::Target::Oper); + + for (size_t i = m_stage_ocr_end_frame; i < m_video_frame_count; i += skip_frames(skip_count) + 1) { + cv::Mat frame; + *m_video_ptr >> frame; + if (frame.empty()) { + Log.error(i, "frame is empty"); + callback(AsstMsg::SubTaskError, basic_info_with_what("MatchDeployment")); + return false; + } + + cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); + + oper_analyzer.set_image(frame); + bool analyzed = oper_analyzer.analyze() && oper_analyzer.get_pause_button(); + show_img(oper_analyzer); + if (analyzed) { + m_battle_start_frame = i; + break; + } + } + const auto& deployment = oper_analyzer.get_opers(); + + auto avatar_task_ptr = Task.get("BattleAvatarDataForFormation"); + for (const auto& [name, formation_avatar] : m_formation) { + BestMatchImageAnalyzer best_match_analyzer(formation_avatar); + best_match_analyzer.set_task_info(avatar_task_ptr); + + std::unordered_set roles = { BattleData.get_role(name) }; + if (name == "阿米娅") { + roles.emplace(battle::Role::Warrior); + } + + // 编队界面，有些视频会有些花里胡哨的特效遮挡啥的，所以尽量减小点模板尺寸 + auto crop_roi = make_rect(avatar_task_ptr->rect_move); + // 小车的缩放太离谱了 + const size_t scale_ends = BattleData.get_rarity(name) == 1 ? 200 : 125; + std::unordered_map candidate; + for (const auto& oper : deployment) { + if (!roles.contains(oper.role)) { + continue; + } + cv::Mat crop_avatar = oper.avatar(crop_roi); + // 从编队到待部署区，每个干员的缩放大小都不一样，暴力跑一遍 + // TODO: 不知道gamedata里有没有这个缩放数据，直接去拿 + for (size_t i = 100; i < scale_ends; ++i) { + double avatar_scale = i / 100.0; + const auto resize_method = avatar_scale < 1.0 ? cv::INTER_AREA : cv::INTER_LINEAR; + cv::Mat resized_avatar; + cv::resize(crop_avatar, resized_avatar, cv::Size(), avatar_scale, avatar_scale, resize_method); + std::string flag = name + "|" + std::to_string(oper.index) + "|" + std::to_string(i); + best_match_analyzer.append_templ(flag, resized_avatar); + candidate.emplace(flag, oper.avatar); + } + } + bool analyzed = best_match_analyzer.analyze(); + // show_img(best_match_analyzer); + if (!analyzed) { + Log.warn(m_battle_start_frame, "failed to match", name); + continue; + } + m_all_avatars.emplace(name, candidate.at(best_match_analyzer.get_result().name)); + } + callback(AsstMsg::SubTaskCompleted, basic_info_with_what("MatchDeployment")); + + return !m_all_avatars.empty(); +} + +bool asst::CombatRecordRecognitionTask::slice_video() +{ + LogTraceFunction; + callback(AsstMsg::SubTaskStart, basic_info_with_what("Slice")); + + const int skip_count = m_video_fps > m_deployment_fps ? static_cast(m_video_fps / m_deployment_fps) - 1 : 0; + + int not_in_battle_count = 0; + bool in_segment = false; + + cv::Mat frame; + cv::Mat pre_frame; + + size_t i = m_battle_start_frame; + size_t ends = m_video_frame_count - skip_count - 10; + + auto battle_over = [&]() { + if (m_clips.empty()) { + return; + } + if (m_battle_end_frame == 0) { + m_battle_end_frame = i; + } + if (!in_segment) { + return; + } + auto& pre_clip = m_clips.back(); + pre_clip.end_frame_index = i - skip_count; + pre_clip.end_frame = pre_frame; + in_segment = false; + }; + for (; i < ends; i += skip_frames(skip_count) + 1, pre_frame = frame) { + *m_video_ptr >> frame; + if (frame.empty()) { + Log.warn(i, "frame is empty"); + battle_over(); + break; + } + cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); + + BattleImageAnalyzer oper_analyzer(frame); + oper_analyzer.set_target(BattleImageAnalyzer::Target::Oper | BattleImageAnalyzer::Target::DetailPage); + bool analyzed = oper_analyzer.analyze(); + show_img(oper_analyzer); + + if (!analyzed) { + battle_over(); + if (++not_in_battle_count > 10) { + break; + } + continue; + } + m_battle_end_frame = 0; + not_in_battle_count = 0; + + const auto& cur_opers = oper_analyzer.get_opers(); + bool continuity = true; + int pre_distance = 0; + for (auto iter = cur_opers.begin(); iter != cur_opers.end(); ++iter) { + if (iter == cur_opers.begin()) { + continue; + } + int distance = std::abs(iter->rect.x - (iter - 1)->rect.x); + if (pre_distance && std::abs(distance - pre_distance) > 5) { + continuity = false; + } + pre_distance = distance; + } + + bool in_detail_page = oper_analyzer.get_in_detail_page() || !oper_analyzer.get_pause_button(); + bool oper_auto_retreat = + in_segment && continuity && !m_clips.empty() && cur_opers.size() != m_clips.back().deployment.size(); + + if (in_detail_page || oper_auto_retreat) { + if (m_clips.empty()) { + continue; + } + auto& pre_clip = m_clips.back(); + if (pre_clip.ends_oper_name.empty()) { + pre_clip.ends_oper_name = analyze_detail_page_oper_name(frame); + } + if (!in_segment) { + continue; + } + pre_clip.end_frame_index = i - skip_count; + pre_clip.end_frame = pre_frame; + in_segment = false; + + continue; + } + else if (!continuity) { + Log.warn(i, "opers is not continuity"); + continue; + } + else if (!in_segment) { + ClipInfo info; + info.start_frame_index = i; // 后处理会加个 offset + info.end_frame_index = i; + info.deployment = cur_opers; + info.start_frame = frame; + m_clips.emplace_back(std::move(info)); + + in_segment = true; + } + } + battle_over(); + + if (m_clips.empty()) { + callback(AsstMsg::SubTaskError, basic_info_with_what("Slice")); + return false; + } + + // post process clips + constexpr int offset_ms = 300; + const int offset_frame = static_cast(offset_ms * m_video_fps / 1000.0); + + for (auto iter = m_clips.begin(); iter != m_clips.end();) { + ClipInfo& clip = *iter; + if (clip.end_frame_index <= clip.start_frame_index) { + Log.warn(__FUNCTION__, "deployment has no changes or frame error", clip.start_frame_index, + clip.end_frame_index); + iter = m_clips.erase(iter); + continue; + } + + size_t new_start = clip.start_frame_index + offset_frame; + if (new_start < clip.end_frame_index) { + clip.start_frame_index = new_start; + } + + if (iter == m_clips.begin()) { + ++iter; + continue; + } + + ClipInfo& pre_clip = *(iter - 1); + bool deployment_changed = false; + if (iter != m_clips.begin() && clip.deployment.size() == pre_clip.deployment.size()) { + for (size_t j = 0; j < clip.deployment.size(); ++j) { + deployment_changed |= clip.deployment[j].role != pre_clip.deployment[j].role; + } + } + else { + deployment_changed = true; + } + iter->deployment_changed = deployment_changed; + ++iter; + } + m_video_ptr->set(cv::CAP_PROP_POS_FRAMES, static_cast(m_battle_start_frame)); + + callback(AsstMsg::SubTaskCompleted, basic_info_with_what("Slice")); + return true; +} + +bool asst::CombatRecordRecognitionTask::analyze_clip(ClipInfo& clip, ClipInfo* pre_clip_ptr) +{ + LogTraceFunction; + + if (!clip.deployment_changed) { + return compare_skill(clip, pre_clip_ptr); + } + + if (!detect_operators(clip, pre_clip_ptr)) { + return false; + } + if (!classify_direction(clip, pre_clip_ptr)) { + return false; + } + if (!process_changes(clip, pre_clip_ptr)) { + return false; + } + + return true; +} + +bool asst::CombatRecordRecognitionTask::compare_skill(ClipInfo& clip, ClipInfo* pre_clip_ptr) +{ + LogTraceFunction; + + if (!pre_clip_ptr) { + Log.error("First clip but deployment not changed"); + return false; + } + callback(AsstMsg::SubTaskStart, basic_info_with_what("CompSkill")); + + const std::string oper_name = pre_clip_ptr->ends_oper_name; + const Point target_location = m_operator_locations[oper_name]; + const Point target_position = m_normal_tile_info[target_location].pos; + BattleSkillReadyImageAnalyzer analyzer(pre_clip_ptr->end_frame); + analyzer.set_base_point(target_position); + bool pre_ready = analyzer.analyze(); + show_img(analyzer); + + if (!pre_ready) { + // TODO: 有可能是点开之后等着技能转好，这种情况比较难处理 + return true; + } + + // 有些干员明明点完技能了，但图标还会持续几百毫秒才消失 + // 所以这里要跳过一段时间 + constexpr int skip_ms = 500; + size_t cls_begin = clip.start_frame_index + static_cast(skip_ms * m_video_fps / 1000.0); + if (cls_begin > clip.end_frame_index) { + Log.warn("skip too much"); + cls_begin = clip.start_frame_index + (clip.end_frame_index - clip.start_frame_index) / 2; + } + const size_t begin_skip = cls_begin - static_cast(m_video_ptr->get(cv::CAP_PROP_POS_FRAMES)); + skip_frames(begin_skip); + + cv::Mat frame; + *m_video_ptr >> frame; + if (frame.empty()) { + Log.error("frame is empty"); + callback(AsstMsg::SubTaskError, basic_info_with_what("CompSkill")); + return false; + } + cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); + analyzer.set_image(frame); + bool cur_ready = analyzer.analyze(); + + if (pre_ready && !cur_ready) { + json::object skill_json { + { "type", "Skill" }, + { "location", json::array { target_location.x, target_location.y } }, + { "name", oper_name }, + }; + Log.info("skill json", skill_json.to_string()); + auto& actions_json = m_copilot_json["actions"].as_array(); + actions_json.emplace_back(std::move(skill_json)); + } + + callback(AsstMsg::SubTaskCompleted, basic_info_with_what("CompSkill")); + return true; +} + +bool asst::CombatRecordRecognitionTask::detect_operators(ClipInfo& clip, [[maybe_unused]] ClipInfo* pre_clip_ptr) +{ + LogTraceFunction; + + callback(AsstMsg::SubTaskStart, basic_info_with_what("DetectOperators")); + + const size_t frame_count = clip.end_frame_index - clip.start_frame_index; + + /* detect operators on the battefield */ + using DetectionResult = std::unordered_set; + std::unordered_map> oper_det_samping; + const Rect det_box_move = Task.get("BattleOperBoxRectMove")->rect_move; + + constexpr size_t OperDetSamplingCount = 20; + const size_t skip_count = + frame_count > (OperDetSamplingCount + 1) ? frame_count / (OperDetSamplingCount + 1) - 1 : 0; + + const size_t det_begin = clip.start_frame_index + skip_count; + const size_t det_end = clip.end_frame_index - skip_count; + + const size_t begin_skip = det_begin - static_cast(m_video_ptr->get(cv::CAP_PROP_POS_FRAMES)); + skip_frames(begin_skip); + + for (size_t i = det_begin; i <= det_end; i += skip_frames(skip_count) + 1) { + cv::Mat frame; + *m_video_ptr >> frame; + if (frame.empty()) { + Log.error(i, "frame is empty"); + callback(AsstMsg::SubTaskError, basic_info_with_what("DetectOperators")); + return false; + } + + cv::resize(frame, frame, cv::Size(), m_scale, m_scale, cv::INTER_AREA); + BattleOperatorsImageAnalyzer analyzer(frame); + analyzer.analyze(); + show_img(analyzer); + + DetectionResult cur_locations; + auto tiles = m_normal_tile_info | views::values; + for (const auto& box : analyzer.get_results()) { + Rect rect = box.rect.move(det_box_move); + auto iter = ranges::find_if(tiles, [&](const TilePack::TileInfo& t) { return rect.include(t.pos); }); + if (iter == tiles.end()) { + Log.warn(i, __FUNCTION__, "no pos", box.rect.to_string(), rect); + continue; + } + cur_locations.emplace((*iter).loc); + } + oper_det_samping[std::move(cur_locations)] += 1; + + clip.random_frames.emplace_back(frame); // for classify_direction + } + + /* 取众数 */ + auto oper_det_iter = ranges::max_element(oper_det_samping, + [&](const auto& lhs, const auto& rhs) { return lhs.second < rhs.second; }); + if (oper_det_iter == oper_det_samping.end()) { + Log.error(__FUNCTION__, "oper_det_samping is empty"); + callback(AsstMsg::SubTaskError, basic_info_with_what("DetectOperators")); + return false; + } + + for (const Point& loc : oper_det_iter->first) { + clip.battlefield.emplace(loc, BattlefiledOper {}); + } + + callback(AsstMsg::SubTaskCompleted, basic_info_with_what("DetectOperators")); + return true; +} + +bool asst::CombatRecordRecognitionTask::classify_direction(ClipInfo& clip, ClipInfo* pre_clip_ptr) +{ + LogTraceFunction; + + if (!pre_clip_ptr) { + Log.info("first clip, skip"); + callback(AsstMsg::SubTaskCompleted, basic_info_with_what("ClassifyDirection")); + return true; + } + + std::vector newcomer; + for (const Point& loc : clip.battlefield | views::keys) { + if (pre_clip_ptr->battlefield.contains(loc)) { + continue; + } + newcomer.emplace_back(loc); + } + if (newcomer.empty()) { + return true; + } + callback(AsstMsg::SubTaskStart, basic_info_with_what("ClassifyDirection")); + + /* classify direction */ + std::unordered_map> dir_cls_sampling; + + for (const cv::Mat& frame : clip.random_frames) { + BattleDeployDirectionImageAnalyzer analyzer(frame); + for (const auto& loc : newcomer) { + analyzer.set_base_point(m_normal_tile_info.at(loc).pos); + analyzer.analyze(); + show_img(analyzer); + auto dir = static_cast(analyzer.get_class_id()); + dir_cls_sampling[loc][dir] += 1; + } + } + + /* 取众数 */ + for (const auto& [loc, sampling] : dir_cls_sampling) { + auto dir_cls_iter = + ranges::max_element(sampling, [&](const auto& lhs, const auto& rhs) { return lhs.second < rhs.second; }); + + if (dir_cls_iter == sampling.end()) { + Log.error(__FUNCTION__, "dir_cls_sampling is empty"); + callback(AsstMsg::SubTaskError, basic_info_with_what("ClassifyDirection")); + return false; + } + clip.battlefield[loc].direction = dir_cls_iter->first; + clip.battlefield[loc].new_here = true; + } + callback(AsstMsg::SubTaskCompleted, basic_info_with_what("ClassifyDirection")); + return true; +} + +bool asst::CombatRecordRecognitionTask::process_changes(ClipInfo& clip, ClipInfo* pre_clip_ptr) +{ + LogTraceFunction; + std::ignore = m_video_ptr; + std::ignore = clip; + + if (!pre_clip_ptr) { + Log.info("first clip, skip"); + return true; + } + + auto& actions_json = m_copilot_json["actions"].as_array(); + + if (pre_clip_ptr->deployment.size() > clip.deployment.size() || + pre_clip_ptr->battlefield.size() < clip.battlefield.size()) { + // 部署 + std::vector deployed; + ananlyze_deployment_names(clip); + ananlyze_deployment_names(*pre_clip_ptr); + for (const auto& pre_oper : pre_clip_ptr->deployment) { + auto iter = ranges::find_if(clip.deployment, [&](const auto& oper) { return oper.name == pre_oper.name; }); + if (iter != clip.deployment.end()) { + continue; + } + deployed.emplace_back(pre_oper.name); + } + Log.info("deployed", deployed); + + if (deployed.empty()) { + Log.warn("Unknown dployed"); + return false; + } + + auto deployed_iter = deployed.begin(); + for (const auto& [loc, oper] : clip.battlefield) { + if (!oper.new_here) { + continue; + } + std::string name = deployed_iter == deployed.end() ? "UnkownDeployed" : *(deployed_iter++); + json::object deploy_json { + { "type", "Deploy" }, + { "name", name }, // 这里正常应该只有一个人，多了就只能抽奖了（ + { "location", json::array { loc.x, loc.y } }, + { "direction", static_cast(oper.direction) }, + }; + Log.info("deploy json", deploy_json.to_string()); + actions_json.emplace_back(std::move(deploy_json)); + + m_operator_locations.insert_or_assign(name, loc); + m_location_operators.insert_or_assign(loc, name); + } + } + else if (pre_clip_ptr->deployment.size() < clip.deployment.size() || + pre_clip_ptr->battlefield.size() > clip.battlefield.size()) { + // 撤退 + for (const auto& [pre_loc, pre_oper] : pre_clip_ptr->battlefield) { + if (clip.battlefield.contains(pre_loc)) { + continue; + } + + std::string name = m_location_operators[pre_loc]; + json::object retreat_json { + { "type", "Retreat" }, + { "location", json::array { pre_loc.x, pre_loc.y } }, + { "name", name }, + }; + Log.info("retreat json", retreat_json.to_string()); + actions_json.emplace_back(std::move(retreat_json)); + + m_location_operators.erase(pre_loc); + m_operator_locations.erase(name); + } + } + else { + Log.warn("Unknown changes, deployment:", pre_clip_ptr->deployment.size(), clip.deployment.size(), + "battlefield:", pre_clip_ptr->battlefield.size(), clip.battlefield.size()); + } + + return true; +} + +void asst::CombatRecordRecognitionTask::ananlyze_deployment_names(ClipInfo& clip) +{ + LogTraceFunction; + + for (auto& oper : clip.deployment) { + if (!oper.name.empty()) { + continue; + } + BestMatchImageAnalyzer avatar_analyzer(oper.avatar); + static const double threshold = Task.get("BattleAvatarDataForVideo")->templ_threshold; + avatar_analyzer.set_threshold(threshold); + // static const double drone_threshold = Task.get("BattleDroneAvatarData")->templ_threshold; + // avatar_analyzer.set_threshold(oper.role == battle::Role::Drone ? drone_threshold : threshold); + + for (const auto& [name, avatar] : m_all_avatars) { + std::unordered_set roles = { BattleData.get_role(name) }; + if (name == "阿米娅") { + roles.emplace(battle::Role::Warrior); + } + if (roles.contains(oper.role)) { + avatar_analyzer.append_templ(name, avatar); + } + } + bool analyzed = avatar_analyzer.analyze(); + // show_img(avatar_analyzer.get_draw()); + if (analyzed) { + oper.name = avatar_analyzer.get_result().name; + } + else { + oper.name = "UnknownDeployment"; + } + } +} + +size_t asst::CombatRecordRecognitionTask::skip_frames(size_t count) +{ + for (size_t i = 0; i < count; ++i) { + cv::Mat ignore; + *m_video_ptr >> ignore; + } + return count; +} + +std::string asst::CombatRecordRecognitionTask::analyze_detail_page_oper_name(const cv::Mat& frame) +{ + auto analyze = [&](OcrImageAnalyzer& name_analyzer) { + name_analyzer.set_image(frame); + name_analyzer.set_task_info("BattleOperName"); + name_analyzer.set_replace(Task.get("CharsNameOcrReplace")->replace_map, + Task.get("CharsNameOcrReplace")->replace_full); + if (!name_analyzer.analyze()) { + return std::string(); + } + name_analyzer.sort_result_by_score(); + return name_analyzer.get_result().front().text; + }; + + OcrWithPreprocessImageAnalyzer preproc_analyzer; + std::string name = analyze(preproc_analyzer); + + if (BattleData.is_name_invalid(name)) { + Log.warn("ocr with preprocess got a invalid name, try to use detect model", name); + OcrImageAnalyzer det_analyzer; + std::string det_name = analyze(det_analyzer); + if (BattleData.is_name_invalid(det_name)) { + return std::string(); + } + Log.info("use ocr with det", det_name); + return det_name; + } + return name; +} + +void asst::CombatRecordRecognitionTask::show_img(const asst::AbstractImageAnalyzer& analyzer) +{ +#ifdef ASST_DEBUG + show_img(analyzer.get_draw()); +#else + std::ignore = analyzer; +#endif +} + +void asst::CombatRecordRecognitionTask::show_img(const cv::Mat& img) +{ +#ifdef ASST_DEBUG + cv::imshow(DrawWindow, img); + cv::waitKey(1); +#else + std::ignore = img; +#endif +} diff --git a/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.h b/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.h new file mode 100644 index 0000000000..c97eaa6e43 --- /dev/null +++ b/src/MaaCore/Task/Experiment/CombatRecordRecognitionTask.h @@ -0,0 +1,101 @@ +#pragma once + +#include "Task/AbstractTask.h" + +#include "Common/AsstBattleDef.h" +#include "Common/AsstTypes.h" +#include "Config/Miscellaneous/TilePack.h" + +#include + +namespace cv +{ + class VideoCapture; +} + +namespace asst +{ + class AbstractImageAnalyzer; + + class CombatRecordRecognitionTask : public AbstractTask + { + public: + using AbstractTask::AbstractTask; + virtual ~CombatRecordRecognitionTask() override = default; + + bool set_video_path(const std::filesystem::path& path); + bool set_stage_name(const std::string& stage_name); + + protected: + virtual bool _run() override; + + private: + struct BattlefiledOper + { + bool new_here = false; + battle::DeployDirection direction = battle::DeployDirection::None; + std::string name; + }; + + struct ClipInfo + { + size_t start_frame_index = 0; + size_t end_frame_index = 0; + cv::Mat start_frame; + cv::Mat end_frame; + std::vector deployment; + bool deployment_changed = true; + std::unordered_map battlefield; + std::vector random_frames; + std::string ends_oper_name; + }; + + bool analyze_formation(); + bool analyze_stage(); + bool analyze_deployment(); + bool slice_video(); + bool analyze_clip(ClipInfo& clip, ClipInfo* pre_clip_ptr = nullptr); + bool compare_skill(ClipInfo& clip, ClipInfo* pre_clip_ptr = nullptr); + bool detect_operators(ClipInfo& clip, ClipInfo* pre_clip_ptr = nullptr); + bool classify_direction(ClipInfo& clip, ClipInfo* pre_clip_ptr = nullptr); + bool process_changes(ClipInfo& clip, ClipInfo* pre_clip_ptr = nullptr); + void ananlyze_deployment_names(ClipInfo& clip); + size_t skip_frames(size_t count); + + static std::string analyze_detail_page_oper_name(const cv::Mat& frame); + + std::filesystem::path m_video_path; + std::shared_ptr m_video_ptr = nullptr; + std::string m_stage_name; + double m_video_fps = 0; + size_t m_video_frame_count = 0; + double m_scale = 0; + + int m_formation_fps = 2; + int m_stage_ocr_fps = 2; + int m_deployment_fps = 5; + + size_t m_formation_end_frame = 0; + size_t m_stage_ocr_end_frame = 0; + size_t m_battle_start_frame = 0; + size_t m_battle_end_frame = 0; + + std::vector m_clips; + + std::unordered_map m_normal_tile_info; + std::unordered_map m_formation; + std::unordered_map m_all_avatars; + + std::unordered_map m_operator_locations; + std::unordered_map m_location_operators; + + json::value m_copilot_json; + + void show_img(const AbstractImageAnalyzer& analyzer); + void show_img(const cv::Mat& img); + +#ifdef ASST_DEBUG + inline static const std::string DrawWindow = "CombatRecord"; +#endif + }; +} // namespace asst diff --git a/src/MaaCore/Task/SingleStep/SingleStepBattleProcessTask.cpp b/src/MaaCore/Task/Experiment/SingleStepBattleProcessTask.cpp similarity index 100% rename from src/MaaCore/Task/SingleStep/SingleStepBattleProcessTask.cpp rename to src/MaaCore/Task/Experiment/SingleStepBattleProcessTask.cpp diff --git a/src/MaaCore/Task/SingleStep/SingleStepBattleProcessTask.h b/src/MaaCore/Task/Experiment/SingleStepBattleProcessTask.h similarity index 100% rename from src/MaaCore/Task/SingleStep/SingleStepBattleProcessTask.h rename to src/MaaCore/Task/Experiment/SingleStepBattleProcessTask.h diff --git a/src/MaaCore/Task/Interface/DebugTask.cpp b/src/MaaCore/Task/Interface/DebugTask.cpp index 6f343af8c3..57f2a71c30 100644 --- a/src/MaaCore/Task/Interface/DebugTask.cpp +++ b/src/MaaCore/Task/Interface/DebugTask.cpp @@ -6,7 +6,7 @@ #include "Utils/ImageIo.hpp" #include "Utils/Logger.hpp" -#include "Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h" +#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" #include "Vision/Miscellaneous/DepotImageAnalyzer.h" #include "Vision/Miscellaneous/StageDropsImageAnalyzer.h" diff --git a/src/MaaCore/Task/Interface/SingleStepTask.cpp b/src/MaaCore/Task/Interface/SingleStepTask.cpp index bc931ee3e7..433ef4dee9 100644 --- a/src/MaaCore/Task/Interface/SingleStepTask.cpp +++ b/src/MaaCore/Task/Interface/SingleStepTask.cpp @@ -1,8 +1,8 @@ #include "SingleStepTask.h" #include "Config/Miscellaneous/CopilotConfig.h" +#include "Task/Experiment/SingleStepBattleProcessTask.h" #include "Task/ProcessTask.h" -#include "Task/SingleStep/SingleStepBattleProcessTask.h" #include "Utils/Logger.hpp" asst::SingleStepTask::SingleStepTask(const AsstCallback& callback, Assistant* inst) diff --git a/src/MaaCore/Task/Interface/VideoRecognitionTask.cpp b/src/MaaCore/Task/Interface/VideoRecognitionTask.cpp new file mode 100644 index 0000000000..b6028ef331 --- /dev/null +++ b/src/MaaCore/Task/Interface/VideoRecognitionTask.cpp @@ -0,0 +1,39 @@ +#include "VideoRecognitionTask.h" + +#include "Task/Experiment/CombatRecordRecognitionTask.h" +#include "Utils/Logger.hpp" + +#include + +asst::VideoRecognitionTask::VideoRecognitionTask(const AsstCallback& callback, Assistant* inst) + : InterfaceTask(callback, inst, TaskType) +{} + +bool asst::VideoRecognitionTask::set_params(const json::value& params) +{ + LogTraceFunction; + + if (m_running) { + return false; + } + + auto filename_opt = params.find("filename"); + if (!filename_opt) { + Log.error("No filename"); + return false; + } + std::string stage_name = params.get("stage", ""); + + auto export_task_ptr = std::make_shared(m_callback, m_inst, TaskType); + export_task_ptr->set_retry_times(0); + bool ret = export_task_ptr->set_video_path(*filename_opt); + ret &= export_task_ptr->set_stage_name(stage_name); + if (!ret) { + return false; + } + + export_task_ptr->set_retry_times(0); + m_subtasks.emplace_back(std::move(export_task_ptr)); + + return true; +} diff --git a/src/MaaCore/Task/Interface/VideoRecognitionTask.h b/src/MaaCore/Task/Interface/VideoRecognitionTask.h new file mode 100644 index 0000000000..6d846a923b --- /dev/null +++ b/src/MaaCore/Task/Interface/VideoRecognitionTask.h @@ -0,0 +1,19 @@ +#pragma once +#include "Task/InterfaceTask.h" +#include "Task/Miscellaneous/StartGameTaskPlugin.h" + +namespace asst +{ + class ProcessTask; + + class VideoRecognitionTask final : public InterfaceTask + { + public: + inline static constexpr std::string_view TaskType = "VideoRecognition"; + + VideoRecognitionTask(const AsstCallback& callback, Assistant* inst); + virtual ~VideoRecognitionTask() override = default; + + bool set_params(const json::value& params) override; + }; +} diff --git a/src/MaaCore/Task/Miscellaneous/BattleProcessTask.cpp b/src/MaaCore/Task/Miscellaneous/BattleProcessTask.cpp index 77b0b819b5..19da04cdf0 100644 --- a/src/MaaCore/Task/Miscellaneous/BattleProcessTask.cpp +++ b/src/MaaCore/Task/Miscellaneous/BattleProcessTask.cpp @@ -17,8 +17,8 @@ #include "Utils/ImageIo.hpp" #include "Utils/Logger.hpp" #include "Vision/MatchImageAnalyzer.h" -#include "Vision/Miscellaneous/BattleImageAnalyzer.h" -#include "Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h" +#include "Vision/Battle/BattleImageAnalyzer.h" +#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" #include "Vision/OcrWithPreprocessImageAnalyzer.h" using namespace asst::battle; diff --git a/src/MaaCore/Task/Miscellaneous/BattleProcessTask.h b/src/MaaCore/Task/Miscellaneous/BattleProcessTask.h index 55156966f5..72af4fae18 100644 --- a/src/MaaCore/Task/Miscellaneous/BattleProcessTask.h +++ b/src/MaaCore/Task/Miscellaneous/BattleProcessTask.h @@ -5,7 +5,7 @@ #include "Common/AsstBattleDef.h" #include "Common/AsstTypes.h" #include "Config/Miscellaneous/TilePack.h" -#include "Vision/Miscellaneous/BattleImageAnalyzer.h" +#include "Vision/Battle/BattleImageAnalyzer.h" namespace asst { diff --git a/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp b/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp index ff665f98b5..ab9f18ff09 100644 --- a/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp +++ b/src/MaaCore/Task/Reclamation/ReclamationBattlePlugin.cpp @@ -9,7 +9,7 @@ #include "Task/ProcessTask.h" #include "Utils/Logger.hpp" #include "Vision/MatchImageAnalyzer.h" -#include "Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h" +#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" #include "Vision/OcrImageAnalyzer.h" using namespace asst; diff --git a/src/MaaCore/Task/Roguelike/RoguelikeBattleTaskPlugin.cpp b/src/MaaCore/Task/Roguelike/RoguelikeBattleTaskPlugin.cpp index 2acd61780c..d11999b441 100644 --- a/src/MaaCore/Task/Roguelike/RoguelikeBattleTaskPlugin.cpp +++ b/src/MaaCore/Task/Roguelike/RoguelikeBattleTaskPlugin.cpp @@ -18,8 +18,8 @@ #include "Utils/ImageIo.hpp" #include "Utils/Logger.hpp" #include "Vision/MatchImageAnalyzer.h" -#include "Vision/Miscellaneous/BattleImageAnalyzer.h" -#include "Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h" +#include "Vision/Battle/BattleImageAnalyzer.h" +#include "Vision/Battle/BattleSkillReadyImageAnalyzer.h" #include "Vision/OcrWithPreprocessImageAnalyzer.h" using namespace asst::battle; diff --git a/src/MaaCore/Task/Roguelike/RoguelikeBattleTaskPlugin.h b/src/MaaCore/Task/Roguelike/RoguelikeBattleTaskPlugin.h index d5accc0b8c..b64da95c72 100644 --- a/src/MaaCore/Task/Roguelike/RoguelikeBattleTaskPlugin.h +++ b/src/MaaCore/Task/Roguelike/RoguelikeBattleTaskPlugin.h @@ -8,7 +8,7 @@ #include "Config/Miscellaneous/TilePack.h" #include "Task/AbstractTaskPlugin.h" #include "Task/BattleHelper.h" -#include "Vision/Miscellaneous/BattleImageAnalyzer.h" +#include "Vision/Battle/BattleImageAnalyzer.h" namespace asst { diff --git a/src/MaaCore/Utils/Algorithm.hpp b/src/MaaCore/Utils/Algorithm.hpp index e4d3948399..cad8a08fc0 100644 --- a/src/MaaCore/Utils/Algorithm.hpp +++ b/src/MaaCore/Utils/Algorithm.hpp @@ -7,7 +7,7 @@ namespace asst::algorithm { - static std::optional> get_char_allocation_for_each_group( + inline static std::optional> get_char_allocation_for_each_group( const std::unordered_map>& group_list, const std::unordered_set& char_set) { diff --git a/src/MaaCore/Utils/Time.hpp b/src/MaaCore/Utils/Time.hpp index ec56684bcc..46a1690b1a 100644 --- a/src/MaaCore/Utils/Time.hpp +++ b/src/MaaCore/Utils/Time.hpp @@ -40,7 +40,7 @@ namespace asst::utils return buff; } - inline std::string get_random_filestem() + inline std::string get_time_filestem() { std::string stem = utils::get_format_time(); string_replace_all_in_place(stem, { { ":", "-" }, { " ", "_" }, { ".", "-" } }); diff --git a/src/MaaCore/Vision/AbstractImageAnalyzer.cpp b/src/MaaCore/Vision/AbstractImageAnalyzer.cpp index b99567aa58..b9108e5a6d 100644 --- a/src/MaaCore/Vision/AbstractImageAnalyzer.cpp +++ b/src/MaaCore/Vision/AbstractImageAnalyzer.cpp @@ -17,6 +17,12 @@ asst::AbstractImageAnalyzer::AbstractImageAnalyzer(const cv::Mat& image) #endif {} +#ifdef ASST_DEBUG +asst::AbstractImageAnalyzer::AbstractImageAnalyzer(const cv::Mat& image, cv::Mat& draw) + : m_image(image), m_roi(correct_rect(Rect(), image)), m_image_draw(draw) +{} +#endif + asst::AbstractImageAnalyzer::AbstractImageAnalyzer(const cv::Mat& image, Assistant* inst) : InstHelper(inst), m_image(image), m_roi(correct_rect(Rect(), image)) #ifdef ASST_DEBUG @@ -33,11 +39,24 @@ void asst::AbstractImageAnalyzer::set_image(const cv::Mat& image) #endif } +#ifdef ASST_DEBUG +void asst::AbstractImageAnalyzer::set_image(const cv::Mat& image, cv::Mat& draw) +{ + m_image = image; + m_image_draw = draw; +} +#endif + void asst::AbstractImageAnalyzer::set_roi(const Rect& roi) noexcept { m_roi = correct_rect(roi, m_image); } +void asst::AbstractImageAnalyzer::set_log_tracing(bool enable) +{ + m_log_tracing = enable; +} + asst::Rect asst::AbstractImageAnalyzer::correct_rect(const Rect& rect, const cv::Mat& image) noexcept { if (image.empty()) { @@ -79,7 +98,7 @@ asst::Rect asst::AbstractImageAnalyzer::correct_rect(const Rect& rect, const cv: bool asst::AbstractImageAnalyzer::save_img(const std::filesystem::path& relative_dir) { - std::string stem = utils::get_random_filestem(); + std::string stem = utils::get_time_filestem(); auto relative_path = relative_dir / (stem + "_raw.png"); Log.trace("Save image", relative_path); bool ret = asst::imwrite(relative_path, m_image); @@ -90,3 +109,10 @@ bool asst::AbstractImageAnalyzer::save_img(const std::filesystem::path& relative return ret; } + +#ifdef ASST_DEBUG +cv::Mat asst::AbstractImageAnalyzer::get_draw() const +{ + return m_image_draw; +} +#endif diff --git a/src/MaaCore/Vision/AbstractImageAnalyzer.h b/src/MaaCore/Vision/AbstractImageAnalyzer.h index d82673d155..8a0c0913ff 100644 --- a/src/MaaCore/Vision/AbstractImageAnalyzer.h +++ b/src/MaaCore/Vision/AbstractImageAnalyzer.h @@ -20,21 +20,32 @@ namespace asst public: AbstractImageAnalyzer() = default; AbstractImageAnalyzer(const cv::Mat& image); +#ifdef ASST_DEBUG + AbstractImageAnalyzer(const cv::Mat& image, cv::Mat& draw); +#endif AbstractImageAnalyzer(const cv::Mat& image, Assistant* inst); AbstractImageAnalyzer(const AbstractImageAnalyzer&) = delete; AbstractImageAnalyzer(AbstractImageAnalyzer&&) = delete; virtual ~AbstractImageAnalyzer() = default; virtual void set_image(const cv::Mat& image); +#ifdef ASST_DEBUG + virtual void set_image(const cv::Mat& image, cv::Mat& draw); +#endif virtual void set_roi(const Rect& roi) noexcept; virtual bool analyze() = 0; + virtual void set_log_tracing(bool enable); AbstractImageAnalyzer& operator=(const AbstractImageAnalyzer&) = delete; AbstractImageAnalyzer& operator=(AbstractImageAnalyzer&&) = delete; bool save_img(const std::filesystem::path& relative_dir = utils::path("debug")); +#ifdef ASST_DEBUG + cv::Mat get_draw() const; +#endif + protected: using InstHelper::status; @@ -43,6 +54,7 @@ namespace asst cv::Mat m_image; Rect m_roi; + bool m_log_tracing = true; #ifdef ASST_DEBUG cv::Mat m_image_draw; diff --git a/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.cpp new file mode 100644 index 0000000000..e9512e0d6b --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.cpp @@ -0,0 +1,69 @@ +#include "BattleDeployDirectionImageAnalyzer.h" + +#include "Utils/NoWarningCV.h" + +#include +#include +#include + +#include + +#include "Config/OnnxSessions.h" +#include "Config/TaskData.h" +#include "Utils/Logger.hpp" + +bool asst::BattleDeployDirectionImageAnalyzer::analyze() +{ + cv::Mat image = m_image(make_rect(m_roi)); + std::vector input = image_to_tensor(image); + + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + constexpr int64_t batch_size = 1; + std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; + + Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, input.data(), input.size(), + input_shape.data(), input_shape.size()); + + constexpr size_t classification_size = 4; + std::array results; + std::array output_shape { batch_size, classification_size }; + Ort::Value output_tensor = Ort::Value::CreateTensor(memory_info, results.data(), results.size(), + output_shape.data(), output_shape.size()); + + auto& session = OnnxSessions::get_instance().get("deploy_direction_cls"); + // 这俩是hardcode在模型里的 + constexpr const char* input_names[] = { "input" }; // session.GetInputName() + constexpr const char* output_names[] = { "output" }; // session.GetOutputName() + + Ort::RunOptions run_options; + session.Run(run_options, input_names, &input_tensor, 1, output_names, &output_tensor, 1); + Log.info(__FUNCTION__, "raw result:", results); + + softmax(results); + Log.info(__FUNCTION__, "after softmax:", results); + + m_class_id = std::max_element(results.begin(), results.end()) - results.begin(); + +#ifdef ASST_DEBUG + static const std::unordered_map ClassNames = { + { 0, "Right" }, + { 1, "Down" }, + { 2, "Left" }, + { 3, "Up" }, + }; + cv::putText(m_image_draw, ClassNames.at(m_class_id), cv::Point(m_roi.x, m_roi.y + m_roi.height), + cv::FONT_HERSHEY_PLAIN, 1.2, cv::Scalar(0, 255, 0), 2); + cv::putText(m_image_draw, std::to_string(results[m_class_id]), cv::Point(m_roi.x, m_roi.y + m_roi.height + 20), + cv::FONT_HERSHEY_PLAIN, 1.2, cv::Scalar(0, 255, 0), 2); +#endif + + return true; +} + +void asst::BattleDeployDirectionImageAnalyzer::set_base_point(const Point& pt) +{ + auto task_ptr = Task.get("BattleDeployDirectionRectMove"); + const Rect& roi_move = task_ptr->rect_move; + + set_roi(Rect(pt.x, pt.y, 0, 0).move(roi_move)); +} diff --git a/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.h new file mode 100644 index 0000000000..730dd498e7 --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattleDeployDirectionImageAnalyzer.h @@ -0,0 +1,22 @@ +#pragma once + +#include "Vision/OnnxRuntimeImageAnalyzer.h" + +namespace asst +{ + class BattleDeployDirectionImageAnalyzer final : public OnnxRuntimeImageAnalyzer + { + public: + using OnnxRuntimeImageAnalyzer::OnnxRuntimeImageAnalyzer; + virtual ~BattleDeployDirectionImageAnalyzer() override = default; + + virtual bool analyze() override; + + size_t get_class_id() const { return m_class_id; } + + void set_base_point(const Point& pt); + + private: + size_t m_class_id = 0; + }; +} diff --git a/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.cpp new file mode 100644 index 0000000000..eb6f01835c --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.cpp @@ -0,0 +1,60 @@ +#include "BattleFormationImageAnalyzer.h" + +#include "Config/Miscellaneous/BattleDataConfig.h" +#include "Config/TaskData.h" +#include "Utils/Logger.hpp" +#include "Utils/NoWarningCV.h" +#include "Vision/Miscellaneous/ProcessTaskImageAnalyzer.h" +#include "Vision/OcrWithFlagTemplImageAnalyzer.h" + +bool asst::BattleFormationImageAnalyzer::analyze() +{ + LogTraceFunction; + m_result.clear(); + + ProcessTaskImageAnalyzer start_button_analyzer(m_image, Task.get("BattleStartAll")->next, nullptr); + if (!start_button_analyzer.analyze()) { + return false; + } + + OcrWithFlagTemplImageAnalyzer oper_names_analyzer(m_image); + oper_names_analyzer.set_task_info("BattleFormationOCRNameFlag", "BattleFormationOperNames"); + oper_names_analyzer.set_expansion(3); + if (!oper_names_analyzer.analyze()) { + return false; + } + + auto proc_ocr_result = [&]() { + const auto& oper_names = oper_names_analyzer.get_result(); + for (const auto& name_res : oper_names) { + if (BattleData.is_name_invalid(name_res.text)) { + continue; + } + const std::string& name = name_res.text; + Rect base_point { name_res.rect.x + name_res.rect.width, name_res.rect.y, 0, 0 }; + Rect avatar_rect = base_point.move(Task.get("BattleFormationOperAvatarMove")->rect_move); + cv::Mat avatar = m_image(make_rect(avatar_rect)); + m_result.emplace(name, avatar); +#ifdef ASST_DEBUG + cv::rectangle(m_image_draw, make_rect(name_res.rect), cv::Scalar(0, 255, 0), 2); + cv::rectangle(m_image_draw, make_rect(avatar_rect), cv::Scalar(0, 0, 255), 2); + cv::putText(m_image_draw, utils::utf8_to_ansi(name), cv::Point(avatar_rect.x, avatar_rect.y - 20), 1, 1.2, + cv::Scalar(0, 0, 255), 2); +#endif + } + }; + + proc_ocr_result(); + + // 识别到了结果，但是确实没有可用的干员名。说明游戏UI可能是老版本的，布局不一样 + if (m_result.empty()) { + oper_names_analyzer.set_task_info("BattleFormationOCRNameFlag", "BattleFormationOperNamesOldVersion"); + oper_names_analyzer.set_expansion(2); + if (!oper_names_analyzer.analyze()) { + return false; + } + proc_ocr_result(); + } + + return !m_result.empty(); +} diff --git a/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.h new file mode 100644 index 0000000000..cbc93c9baf --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattleFormationImageAnalyzer.h @@ -0,0 +1,20 @@ +#pragma once + +#include "Vision/AbstractImageAnalyzer.h" + +namespace asst +{ + class BattleFormationImageAnalyzer : public AbstractImageAnalyzer + { + public: + using AbstractImageAnalyzer::AbstractImageAnalyzer; + virtual ~BattleFormationImageAnalyzer() override = default; + + virtual bool analyze() override; + + const auto& get_result() const { return m_result; } + + private: + std::unordered_map m_result; + }; +} diff --git a/src/MaaCore/Vision/Miscellaneous/BattleImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleImageAnalyzer.cpp similarity index 78% rename from src/MaaCore/Vision/Miscellaneous/BattleImageAnalyzer.cpp rename to src/MaaCore/Vision/Battle/BattleImageAnalyzer.cpp index fa6fdf01b4..d89237f66c 100644 --- a/src/MaaCore/Vision/Miscellaneous/BattleImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Battle/BattleImageAnalyzer.cpp @@ -28,22 +28,26 @@ bool asst::BattleImageAnalyzer::analyze() { clear(); - // HP 作为 flag，无论如何都识别。表明当前画面是在战斗场景的 + // flag 无论如何都识别。表明当前画面是在战斗场景的 bool ret = flag_analyze(); - - if (m_target & Target::Home) { - ret |= home_analyze(); - } - if (!ret) { return false; } + // + // if (m_target & Target::Home) { + // ret |= home_analyze(); + //} + // 可能没有干员（全上场了），所以干员识别结果不影响返回值 if (m_target & Target::Oper) { opers_analyze(); } + if (m_target & Target::DetailPage) { + detail_page_analyze(); + } + if (m_target & Target::Kills) { ret &= kills_analyze(); } @@ -52,9 +56,9 @@ bool asst::BattleImageAnalyzer::analyze() ret &= cost_analyze(); } - if (m_target & Target::Vacancies) { - ret &= vacancies_analyze(); - } + // if (m_target & Target::Vacancies) { + // ret &= vacancies_analyze(); + // } return ret; } @@ -89,6 +93,16 @@ int asst::BattleImageAnalyzer::get_cost() const noexcept return m_cost; } +bool asst::BattleImageAnalyzer::get_in_detail_page() const noexcept +{ + return m_in_detail_page; +} + +bool asst::BattleImageAnalyzer::get_pause_button() const noexcept +{ + return m_pause_button; +} + void asst::BattleImageAnalyzer::clear() noexcept { m_opers.clear(); @@ -108,11 +122,19 @@ void asst::BattleImageAnalyzer::sort_opers_by_cost() bool asst::BattleImageAnalyzer::opers_analyze() { MultiMatchImageAnalyzer flags_analyzer(m_image); - flags_analyzer.set_task_info("BattleOpersFlag"); + const auto& flag_task_ptr = Task.get("BattleOpersFlag"); + flags_analyzer.set_task_info(flag_task_ptr); + flags_analyzer.set_log_tracing(false); + // if (m_target & Target::OperSeleted) { + // // 更大的范围，能把被点击的升起来的干员也识别出来 + // // 但是可能造成误识别 + // flags_analyzer.set_roi(flag_task_ptr->rect_move); + // } if (!flags_analyzer.analyze()) { return false; } flags_analyzer.sort_result_horizontal(); + const auto& flags = flags_analyzer.get_result(); const auto click_move = Task.get("BattleOperClickRange")->rect_move; const auto role_move = Task.get("BattleOperRoleRange")->rect_move; @@ -120,9 +142,10 @@ bool asst::BattleImageAnalyzer::opers_analyze() const auto avlb_move = Task.get("BattleOperAvailable")->rect_move; const auto cooling_move = Task.get("BattleOperCooling")->rect_move; const auto avatar_move = Task.get("BattleOperAvatar")->rect_move; + // const int unselected_y = flag_task_ptr->roi.y; size_t index = 0; - for (const MatchRect& flag_mrect : flags_analyzer.get_result()) { + for (const MatchRect& flag_mrect : flags) { battle::DeploymentOper oper; oper.rect = flag_mrect.rect.move(click_move); if (oper.rect.x + oper.rect.width >= m_image.cols) { @@ -155,10 +178,11 @@ bool asst::BattleImageAnalyzer::opers_analyze() Rect role_rect = flag_mrect.rect.move(role_move); oper.role = oper_role_analyze(role_rect); - // 费用识别的不太准，暂时也没用上，先注释掉，TODO：优化费用识别 + // 干员费用识别的不太准，暂时也没用上，先注释掉，TODO：优化费用识别 // Rect cost_rect = flag_mrect.rect.move(cost_move); // oper.cost = oper_cost_analyze(cost_rect); oper.index = index++; + // oper.selected = flag_mrect.rect.y < unselected_y; m_opers.emplace_back(std::move(oper)); } @@ -177,6 +201,7 @@ asst::battle::Role asst::BattleImageAnalyzer::oper_role_analyze(const Rect& roi) static const std::string TaskName = "BattleOperRole"; static const std::string Ext = ".png"; BestMatchImageAnalyzer role_analyzer(m_image); + role_analyzer.set_log_tracing(false); role_analyzer.set_task_info(TaskName); role_analyzer.set_roi(roi); @@ -187,7 +212,7 @@ asst::battle::Role asst::BattleImageAnalyzer::oper_role_analyze(const Rect& roi) return battle::Role::Unknown; } - const auto& templ_name = role_analyzer.get_result_name(); + const auto& templ_name = role_analyzer.get_result().name; std::string role_name = templ_name.substr(TaskName.size(), templ_name.size() - TaskName.size() - Ext.size()); @@ -290,7 +315,7 @@ bool asst::BattleImageAnalyzer::home_analyze() return true; } -bool asst::BattleImageAnalyzer::hp_analyze() +bool asst::BattleImageAnalyzer::hp_flag_analyze() { // 识别 HP 的那个蓝白色图标 auto flag_task_ptr = Task.get("BattleHpFlag"); @@ -307,6 +332,13 @@ bool asst::BattleImageAnalyzer::hp_analyze() return true; } +bool asst::BattleImageAnalyzer::kills_flag_analyze() +{ + MatchImageAnalyzer flag_analyzer(m_image); + flag_analyzer.set_task_info("BattleKillsFlag"); + return flag_analyzer.analyze(); +} + bool asst::BattleImageAnalyzer::kills_analyze() { OcrWithFlagTemplImageAnalyzer kills_analyzer(m_image); @@ -399,16 +431,47 @@ bool asst::BattleImageAnalyzer::vacancies_analyze() bool asst::BattleImageAnalyzer::flag_analyze() { - MatchImageAnalyzer flag_analyzer(m_image); - flag_analyzer.set_task_info("BattleOfficiallyBegin"); - if (flag_analyzer.analyze()) { - return true; - } - - flag_analyzer.set_task_info("BattleKillsFlag"); - if (flag_analyzer.analyze()) { - return true; - } - - return hp_analyze(); + return pause_button_analyze() || hp_flag_analyze() || kills_flag_analyze(); +} + +bool asst::BattleImageAnalyzer::pause_button_analyze() +{ + auto has_started_task_ptr = Task.get("BattleHasStarted"); + cv::Mat roi = m_image(make_rect(has_started_task_ptr->roi)); + cv::Mat roi_gray; + cv::cvtColor(roi, roi_gray, cv::COLOR_BGR2GRAY); + cv::Mat bin; + const int value_threshold = has_started_task_ptr->special_params[0]; + cv::threshold(roi_gray, bin, value_threshold, 255, cv::THRESH_BINARY); + int count = cv::countNonZero(bin); + const int count_threshold = has_started_task_ptr->special_params[1]; + Log.trace(__FUNCTION__, "count", count, "threshold", count_threshold); + + m_pause_button = count > count_threshold; + return m_pause_button; +} + +bool asst::BattleImageAnalyzer::detail_page_analyze() +{ + auto analyze = [&](const std::string& task_name) { + auto task_ptr = Task.get(task_name); + cv::Mat roi = m_image(make_rect(task_ptr->roi)); + cv::Mat roi_hsv; + cv::cvtColor(roi, roi_hsv, cv::COLOR_BGR2HSV); + cv::Mat bin1; + cv::inRange(roi_hsv, cv::Scalar(99, 235, 235), cv::Scalar(105, 255, 255), bin1); + int count1 = cv::countNonZero(bin1); + + cv::Mat bin2; + cv::inRange(roi_hsv, cv::Scalar(99, 235, 135), cv::Scalar(105, 255, 155), bin2); + int count2 = cv::countNonZero(bin2); + + const int threshold = task_ptr->special_params[0]; + Log.info("detail_page, count:", count1, count2, ", threshold:", threshold); + + return count1 > threshold || count2 > threshold; + }; + + m_in_detail_page = analyze("BattleOperDetailPageFlag") || analyze("BattleOperDetailPageOldFlag"); + return m_in_detail_page; } diff --git a/src/MaaCore/Vision/Miscellaneous/BattleImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleImageAnalyzer.h similarity index 78% rename from src/MaaCore/Vision/Miscellaneous/BattleImageAnalyzer.h rename to src/MaaCore/Vision/Battle/BattleImageAnalyzer.h index f39d551025..272662a5a8 100644 --- a/src/MaaCore/Vision/Miscellaneous/BattleImageAnalyzer.h +++ b/src/MaaCore/Vision/Battle/BattleImageAnalyzer.h @@ -11,13 +11,10 @@ namespace asst enum Target // 需要识别的目标 { None = 0, - HP = 1, // 剩余生命值 - Home = 2, // 蓝色的家门 - Oper = 4, // 下方的干员信息 - // Skill = 8, // cd 转好了可以使用的技能 - Kills = 16, // 击杀数 - Cost = 32, // 费用 - Vacancies = 64, // 剩余可部署干员数 + Oper = 1, // 下方的干员信息 + DetailPage = 2, // 是否点开了详情页 + Kills = 4, // 击杀数 + Cost = 8, // 费用 }; public: @@ -35,12 +32,15 @@ namespace asst int get_kills() const noexcept; int get_total_kills() const noexcept; int get_cost() const noexcept; + bool get_in_detail_page() const noexcept; + bool get_pause_button() const noexcept; void clear() noexcept; void sort_opers_by_cost(); // 高费在前，费用降序 protected: - bool opers_analyze(); // 识别干员 + bool opers_analyze(); // 识别干员 + bool detail_page_analyze(); // 识别是否在详情页 battle::Role oper_role_analyze(const Rect& roi); bool oper_cooling_analyze(const Rect& roi); int oper_cost_analyze(const Rect& roi); @@ -48,11 +48,13 @@ namespace asst bool home_analyze(); // 识别蓝色的家门 // bool skill_analyze(); // 识别技能是否可用 - bool hp_analyze(); // 识别剩余生命值 bool kills_analyze(); // 识别击杀数 bool cost_analyze(); // 识别费用 bool vacancies_analyze(); // 识别剩余可部署人数 - bool flag_analyze(); // 识别暂停按钮 + bool flag_analyze(); + bool hp_flag_analyze(); + bool kills_flag_analyze(); + bool pause_button_analyze(); int m_target = 0; // 待识别的目标 int m_pre_total_kills = 0; // 之前的击杀总数，因为击杀数经常识别不准所以依赖外部传入作为参考 @@ -65,5 +67,7 @@ namespace asst int m_kills = 0; // 击杀数 int m_total_kills = 0; // 击杀总数 int m_cost = 0; // 部署费用 + bool m_in_detail_page = false; + bool m_pause_button = false; }; } // namespace asst diff --git a/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.cpp new file mode 100644 index 0000000000..0dfc840637 --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.cpp @@ -0,0 +1,121 @@ +#include "BattleOperatorsImageAnalyzer.h" + +#include +#include +#include + +#include "Utils/NoWarningCV.h" +#include + +#include "Config/Miscellaneous/TilePack.h" +#include "Config/OnnxSessions.h" +#include "Config/TaskData.h" +#include "Utils/Logger.hpp" + +bool asst::BattleOperatorsImageAnalyzer::analyze() +{ + LogTraceFunction; + + m_results.clear(); + + double x_scale = 640.0 / m_image.cols; + double y_scale = 640.0 / m_image.rows; + + cv::Mat image; + cv::resize(m_image, image, cv::Size(), x_scale, y_scale, cv::INTER_AREA); + std::vector input = image_to_tensor(image); + + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + constexpr int64_t batch_size = 1; + std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; + + Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, input.data(), input.size(), + input_shape.data(), input_shape.size()); + + auto& session = OnnxSessions::get_instance().get("operators_det"); + + Ort::AllocatorWithDefaultOptions allocator; + std::vector input_names; + std::vector output_names; + input_names.emplace_back(session.GetInputName(0, allocator)); + output_names.emplace_back(session.GetOutputName(0, allocator)); + + Ort::RunOptions run_options; + auto outout_tensors = session.Run(run_options, input_names.data(), &input_tensor, 1, output_names.data(), 1); + + const float* raw_output = outout_tensors[0].GetTensorData(); + // output_shape is { 1, 5, 8400 } + std::vector output_shape = outout_tensors[0].GetTensorTypeAndShapeInfo().GetShape(); + + // yolov8 的 onnx 输出和前面的 v5, v7 等似乎不太一样，目前网上 yolov8 的 demo 较少，文档也没找到 + // 这里的输出解析是我跟着数据推测的： + // center_x0, center_x1, ..... center_x8399 + // center_y0, center_y1, ..... center_y8399 + // w0, w1, ..... w8399 + // h0, h1, ..... h8399 + // conf0, conf1, ..... conf8399 + // 如果后面要做多分类，可能得再看下怎么改（我也不知道shape会变成啥样） + std::vector> output(output_shape[1]); + for (int64_t i = 0; i < output_shape[1]; i++) { + output[i] = std::vector(raw_output + i * output_shape[2], raw_output + (i + 1) * output_shape[2]); + } + + std::vector all_results; + const auto& conf_vec = output.back(); + for (size_t i = 0; i < conf_vec.size(); ++i) { + float score = conf_vec[i]; + constexpr float Threshold = 0.3f; + if (score < Threshold) { + continue; + } + int center_x = static_cast(output[0][i] / x_scale); + int center_y = static_cast(output[1][i] / y_scale); + int w = static_cast(output[2][i] / x_scale); + int h = static_cast(output[3][i] / y_scale); + + int x = center_x - w / 2; + int y = center_y - h / 2; + Rect rect { x, y, w, h }; + all_results.emplace_back(Box { Cls::Operator, rect, score }); + } + + // NMS + constexpr double NmsThreshold = 0.7f; + std::sort(all_results.begin(), all_results.end(), [](const Box& a, const Box& b) { return a.score > b.score; }); + std::vector nms_results; + for (size_t i = 0; i < all_results.size(); ++i) { + const auto& box = all_results[i]; + if (box.score < 0.1f) { + continue; + } + nms_results.emplace_back(box); + for (size_t j = i + 1; j < all_results.size(); ++j) { + auto& box2 = all_results[j]; + if (box2.score < 0.1f) { + continue; + } + int iou_area = (make_rect(box.rect) & make_rect(box2.rect)).area(); + if (iou_area > NmsThreshold * box2.rect.area()) { + box2.score = 0; + } + } + } + +#ifdef ASST_DEBUG + int draw_offset_y = static_cast(m_image.rows * -0.15); + int draw_offset_h = static_cast(m_image.rows * 0.13); + for (const auto& box : nms_results) { + Rect draw_rect = box.rect; + draw_rect.y += draw_offset_y; + draw_rect.height += draw_offset_h; + cv::rectangle(m_image_draw, make_rect(draw_rect), cv::Scalar(0, 0, 255), 5); + cv::putText(m_image_draw, std::to_string(box.score), cv::Point(draw_rect.x, draw_rect.y - 10), + cv::FONT_HERSHEY_PLAIN, 1.2, cv::Scalar(0, 0, 255), 2); + m_draw_rect.emplace_back(draw_rect); + } +#endif + + m_results = std::move(nms_results); + + return true; +} diff --git a/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.h new file mode 100644 index 0000000000..88f3ad3414 --- /dev/null +++ b/src/MaaCore/Vision/Battle/BattleOperatorsImageAnalyzer.h @@ -0,0 +1,40 @@ +#pragma once + +#include "Vision/OnnxRuntimeImageAnalyzer.h" + +namespace asst +{ + // 战斗中干员（血条）检测，yolov8 模型 + class BattleOperatorsImageAnalyzer final : public OnnxRuntimeImageAnalyzer + { + public: + enum class Cls + { + Operator = 0, + }; + struct Box + { + Cls cls = Cls::Operator; + Rect rect; + float score = .0f; + }; + + public: + using OnnxRuntimeImageAnalyzer::OnnxRuntimeImageAnalyzer; + virtual ~BattleOperatorsImageAnalyzer() override = default; + + virtual bool analyze() override; + + const std::vector& get_results() const { return m_results; } +#ifdef ASST_DEBUG + const std::vector& get_draw_rect() const { return m_draw_rect; } +#endif + + private: + std::vector m_results; + +#ifdef ASST_DEBUG + std::vector m_draw_rect; +#endif + }; +} diff --git a/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.cpp b/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.cpp similarity index 51% rename from src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.cpp rename to src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.cpp index a4b4fef410..1a8466a587 100644 --- a/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.cpp @@ -8,49 +8,21 @@ #include -#include "Config/OnnxSession.h" +#include "Config/OnnxSessions.h" #include "Config/TaskData.h" #include "Utils/Logger.hpp" -template -static void softmax(T& input) -{ - float rowmax = *std::max_element(input.begin(), input.end()); - std::vector y(input.size()); - float sum = 0.0f; - for (size_t i = 0; i != input.size(); ++i) { - sum += y[i] = std::exp(input[i] - rowmax); - } - for (size_t i = 0; i != input.size(); ++i) { - input[i] = y[i] / sum; - } -} - bool asst::BattleSkillReadyImageAnalyzer::analyze() { - cv::Mat image = m_image(make_rect(m_roi)).clone(); - cv::cvtColor(image, image, cv::COLOR_BGR2RGB); - - size_t input_size = 1ULL * image.cols * image.rows * image.channels(); - std::vector input(input_size); - std::vector output; - size_t count = 0; - for (int k = 0; k < image.channels(); k++) { - for (int i = 0; i < image.rows; i++) { - for (int j = 0; j < image.cols; j++) { - float value = static_cast(image.at(i, j)[k]); - value /= 255; - input[count++] = value; - } - } - } + cv::Mat image = m_image(make_rect(m_roi)); + std::vector input = image_to_tensor(image); auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); constexpr int64_t batch_size = 1; std::array input_shape { batch_size, image.channels(), image.cols, image.rows }; - Ort::Value input_tensor = - Ort::Value::CreateTensor(memory_info, input.data(), input_size, input_shape.data(), input_shape.size()); + Ort::Value input_tensor = Ort::Value::CreateTensor(memory_info, input.data(), input.size(), + input_shape.data(), input_shape.size()); constexpr size_t classification_size = 2; std::array results; @@ -58,18 +30,28 @@ bool asst::BattleSkillReadyImageAnalyzer::analyze() Ort::Value output_tensor = Ort::Value::CreateTensor(memory_info, results.data(), results.size(), output_shape.data(), output_shape.size()); - auto& session = OnnxSession::get_instance().get("skill_ready_rec"); + auto& session = OnnxSessions::get_instance().get("skill_ready_cls"); // 这俩是hardcode在模型里的 constexpr const char* input_names[] = { "input" }; // session.GetInputName() constexpr const char* output_names[] = { "output" }; // session.GetOutputName() Ort::RunOptions run_options; session.Run(run_options, input_names, &input_tensor, 1, output_names, &output_tensor, 1); - Log.info(__FUNCTION__, "raw result, 0: ", results[0], ", 1: ", results[1]); + Log.info(__FUNCTION__, "raw result: ", results); softmax(results); - Log.info(__FUNCTION__, "after softmax, 0: ", results[0], ", 1: ", results[1]); - return results[1] > results[0]; + Log.info(__FUNCTION__, "after softmax: ", results); + bool ready = results[1] > results[0]; + +#ifdef ASST_DEBUG + if (ready) { + cv::rectangle(m_image_draw, make_rect(m_roi), cv::Scalar(0, 165, 255), 2); + cv::putText(m_image_draw, std::to_string(results[1]), cv::Point(m_roi.x, m_roi.y - 10), 1, 1.2, + cv::Scalar(0, 165, 255), 2); + } +#endif + + return ready; } void asst::BattleSkillReadyImageAnalyzer::set_base_point(const Point& pt) diff --git a/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h b/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.h similarity index 53% rename from src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h rename to src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.h index 1ff0894a43..17dc058fd9 100644 --- a/src/MaaCore/Vision/Miscellaneous/BattleSkillReadyImageAnalyzer.h +++ b/src/MaaCore/Vision/Battle/BattleSkillReadyImageAnalyzer.h @@ -1,13 +1,13 @@ #pragma once -#include "Vision/AbstractImageAnalyzer.h" +#include "Vision/OnnxRuntimeImageAnalyzer.h" namespace asst { - class BattleSkillReadyImageAnalyzer final : public AbstractImageAnalyzer + class BattleSkillReadyImageAnalyzer final : public OnnxRuntimeImageAnalyzer { public: - using AbstractImageAnalyzer::AbstractImageAnalyzer; + using OnnxRuntimeImageAnalyzer::OnnxRuntimeImageAnalyzer; virtual ~BattleSkillReadyImageAnalyzer() override = default; virtual bool analyze() override; diff --git a/src/MaaCore/Vision/BestMatchImageAnalyzer.cpp b/src/MaaCore/Vision/BestMatchImageAnalyzer.cpp index c3f33329ae..e14480f104 100644 --- a/src/MaaCore/Vision/BestMatchImageAnalyzer.cpp +++ b/src/MaaCore/Vision/BestMatchImageAnalyzer.cpp @@ -9,11 +9,12 @@ bool asst::BestMatchImageAnalyzer::analyze() { - set_log_tracing(false); + MatchImageAnalyzer::set_log_tracing(false); set_use_cache(false); MatchRect best_matched; - for (const auto& [name, templ] : m_templs) { + for (const auto& templ_info : m_templs) { + auto&& [name, templ] = templ_info; if (templ.empty()) { set_templ_name(name); } @@ -27,13 +28,19 @@ bool asst::BestMatchImageAnalyzer::analyze() const auto& cur_matched = MatchImageAnalyzer::get_result(); if (best_matched.score < cur_matched.score) { best_matched = cur_matched; - m_result_name = name; + m_result = templ_info; } } - m_result = best_matched; - Log.trace("The best match is", best_matched.to_string(), m_result_name); - return m_result.score > 0; + if (m_best_log_tracing) { + Log.trace("The best match is", best_matched.to_string(), m_result.name); + } + return best_matched.score > 0; +} + +void asst::BestMatchImageAnalyzer::set_log_tracing(bool enable) +{ + m_best_log_tracing = enable; } void asst::BestMatchImageAnalyzer::append_templ(std::string name, const cv::Mat& templ) diff --git a/src/MaaCore/Vision/BestMatchImageAnalyzer.h b/src/MaaCore/Vision/BestMatchImageAnalyzer.h index 64ab37e7f1..8480ea7627 100644 --- a/src/MaaCore/Vision/BestMatchImageAnalyzer.h +++ b/src/MaaCore/Vision/BestMatchImageAnalyzer.h @@ -6,14 +6,22 @@ namespace asst { class BestMatchImageAnalyzer : public MatchImageAnalyzer { + public: + struct TemplInfo + { + std::string name; + cv::Mat templ; + }; + public: using MatchImageAnalyzer::MatchImageAnalyzer; virtual ~BestMatchImageAnalyzer() override = default; virtual bool analyze() override; + virtual void set_log_tracing(bool enable) override; void append_templ(std::string name, const cv::Mat& templ = cv::Mat()); - const std::string& get_result_name() const noexcept { return m_result_name; } + const TemplInfo& get_result() const noexcept { return m_result; } private: using MatchImageAnalyzer::set_region_of_appeared; @@ -21,12 +29,8 @@ namespace asst using MatchImageAnalyzer::set_templ_name; using MatchImageAnalyzer::set_use_cache; - struct TemplInfo - { - std::string name; - cv::Mat templ; - }; std::vector m_templs; - std::string m_result_name; + TemplInfo m_result; + bool m_best_log_tracing = true; }; } diff --git a/src/MaaCore/Vision/MatchImageAnalyzer.cpp b/src/MaaCore/Vision/MatchImageAnalyzer.cpp index 5c530751d0..ac738fbc93 100644 --- a/src/MaaCore/Vision/MatchImageAnalyzer.cpp +++ b/src/MaaCore/Vision/MatchImageAnalyzer.cpp @@ -81,11 +81,6 @@ void asst::MatchImageAnalyzer::set_mask_with_close(int with_close) noexcept m_mask_with_close = with_close; } -void asst::MatchImageAnalyzer::set_log_tracing(bool enable) noexcept -{ - m_log_tracing = enable; -} - const asst::MatchRect& asst::MatchImageAnalyzer::get_result() const noexcept { return m_result; diff --git a/src/MaaCore/Vision/MatchImageAnalyzer.h b/src/MaaCore/Vision/MatchImageAnalyzer.h index 8840324e1e..d35db08937 100644 --- a/src/MaaCore/Vision/MatchImageAnalyzer.h +++ b/src/MaaCore/Vision/MatchImageAnalyzer.h @@ -22,7 +22,6 @@ namespace asst void set_task_info(const std::string& task_name); void set_region_of_appeared(Rect region) noexcept; void set_mask_with_close(int with_close) noexcept; - void set_log_tracing(bool enable) noexcept; const MatchRect& get_result() const noexcept; @@ -39,6 +38,5 @@ namespace asst std::pair m_mask_range; bool m_mask_with_src = false; bool m_mask_with_close = false; - bool m_log_tracing = true; }; } diff --git a/src/MaaCore/Vision/Miscellaneous/ProcessTaskImageAnalyzer.cpp b/src/MaaCore/Vision/Miscellaneous/ProcessTaskImageAnalyzer.cpp index 1810773fe6..42861ebbad 100644 --- a/src/MaaCore/Vision/Miscellaneous/ProcessTaskImageAnalyzer.cpp +++ b/src/MaaCore/Vision/Miscellaneous/ProcessTaskImageAnalyzer.cpp @@ -30,15 +30,19 @@ bool asst::ProcessTaskImageAnalyzer::match_analyze(const std::shared_ptrset_region_of_appeared(Rect()); m_match_analyzer->set_task_info(match_task_ptr); - auto region_opt = status()->get_rect(match_task_ptr->name); - if (region_opt) { - m_match_analyzer->set_region_of_appeared(region_opt.value()); + if (m_inst) { + auto region_opt = status()->get_rect(match_task_ptr->name); + if (region_opt) { + m_match_analyzer->set_region_of_appeared(region_opt.value()); + } } if (m_match_analyzer->analyze()) { m_result = match_task_ptr; m_result_rect = m_match_analyzer->get_result().rect; - status()->set_rect(match_task_ptr->name, m_result_rect); + if (m_inst) { + status()->set_rect(match_task_ptr->name, m_result_rect); + } return true; } return false; @@ -87,8 +91,9 @@ bool asst::ProcessTaskImageAnalyzer::ocr_analyze(const std::shared_ptr } analyzer_ptr = &m_ocr_analyzer; } - - (*analyzer_ptr)->set_region_of_appeared(status()->get_rect(ocr_task_ptr->name).value_or(Rect())); + if (m_inst) { + (*analyzer_ptr)->set_region_of_appeared(status()->get_rect(ocr_task_ptr->name).value_or(Rect())); + } (*analyzer_ptr)->set_task_info(ocr_task_ptr); bool ret = (*analyzer_ptr)->analyze(); @@ -99,7 +104,9 @@ bool asst::ProcessTaskImageAnalyzer::ocr_analyze(const std::shared_ptr auto& res = ocr_result.front(); m_result = ocr_task_ptr; m_result_rect = res.rect; - status()->set_rect(ocr_task_ptr->name, m_result_rect); + if (m_inst) { + status()->set_rect(ocr_task_ptr->name, m_result_rect); + } // m_ocr_cache.insert(m_ocr_cache.end(), ocr_result.begin(), ocr_result.end()); Log.trace(__FUNCTION__, "| found", res); } diff --git a/src/MaaCore/Vision/MultiMatchImageAnalyzer.cpp b/src/MaaCore/Vision/MultiMatchImageAnalyzer.cpp index a2e1bcf0c2..7655079a2a 100644 --- a/src/MaaCore/Vision/MultiMatchImageAnalyzer.cpp +++ b/src/MaaCore/Vision/MultiMatchImageAnalyzer.cpp @@ -15,7 +15,9 @@ asst::MultiMatchImageAnalyzer::MultiMatchImageAnalyzer(const cv::Mat& image, std bool asst::MultiMatchImageAnalyzer::analyze() { - Log.trace("MultiMatchImageAnalyzer::analyze | ", m_templ_name); + if (m_log_tracing) { + Log.trace("MultiMatchImageAnalyzer::analyze | ", m_templ_name); + } m_result.clear(); const cv::Mat templ = TemplResource::get_instance().get_templ(m_templ_name); @@ -156,11 +158,9 @@ bool asst::MultiMatchImageAnalyzer::multi_match_templ(const cv::Mat templ) } #endif - Log.trace("multi_match_templ | ", m_templ_name, "result:", m_result, "roi:", m_roi); - - if (!m_result.empty()) { - return true; + if (m_log_tracing) { + Log.trace("multi_match_templ | ", m_templ_name, "result:", m_result, "roi:", m_roi); } - return false; + return !m_result.empty(); } diff --git a/src/MaaCore/Vision/OnnxRuntimeImageAnalyzer.cpp b/src/MaaCore/Vision/OnnxRuntimeImageAnalyzer.cpp new file mode 100644 index 0000000000..042df9d387 --- /dev/null +++ b/src/MaaCore/Vision/OnnxRuntimeImageAnalyzer.cpp @@ -0,0 +1,37 @@ +#include "OnnxRuntimeImageAnalyzer.h" + +#include "Utils/NoWarningCV.h" + +std::vector asst::OnnxRuntimeImageAnalyzer::image_to_tensor(const cv::Mat& image) +{ + cv::Mat src = image.clone(); + cv::cvtColor(src, src, cv::COLOR_BGR2RGB); + + cv::Mat chw = hwc_to_chw(src); + cv::Mat chw_32f; + chw.convertTo(chw_32f, CV_32F, 1.0 / 255.0); + + size_t tensor_size = 1ULL * src.cols * src.rows * src.channels(); + std::vector tensor(tensor_size); + std::memcpy(tensor.data(), chw_32f.data, tensor_size * sizeof(float)); + return tensor; +} + +cv::Mat asst::OnnxRuntimeImageAnalyzer::hwc_to_chw(const cv::Mat& src) +{ + std::vector rgb_images; + cv::split(src, rgb_images); + + // Stretch one-channel images to vector + cv::Mat m_flat_r = rgb_images[0].reshape(1, 1); + cv::Mat m_flat_g = rgb_images[1].reshape(1, 1); + cv::Mat m_flat_b = rgb_images[2].reshape(1, 1); + + // Now we can rearrange channels if need + cv::Mat matArray[] = { m_flat_r, m_flat_g, m_flat_b }; + + cv::Mat flat_image; + // Concatenate three vectors to one + cv::hconcat(matArray, 3, flat_image); + return flat_image; +} diff --git a/src/MaaCore/Vision/OnnxRuntimeImageAnalyzer.h b/src/MaaCore/Vision/OnnxRuntimeImageAnalyzer.h new file mode 100644 index 0000000000..5423b4469e --- /dev/null +++ b/src/MaaCore/Vision/OnnxRuntimeImageAnalyzer.h @@ -0,0 +1,33 @@ +#pragma once + +#include "Vision/AbstractImageAnalyzer.h" + +namespace asst +{ + class OnnxRuntimeImageAnalyzer : public AbstractImageAnalyzer + { + public: + using AbstractImageAnalyzer::AbstractImageAnalyzer; + virtual ~OnnxRuntimeImageAnalyzer() override = default; + + protected: + template + inline static void softmax(T& input) + { + float rowmax = *std::max_element(input.begin(), input.end()); + std::vector y(input.size()); + float sum = 0.0f; + for (size_t i = 0; i != input.size(); ++i) { + sum += y[i] = std::exp(input[i] - rowmax); + } + for (size_t i = 0; i != input.size(); ++i) { + input[i] = y[i] / sum; + } + } + + static std::vector image_to_tensor(const cv::Mat& image); + + private: + static cv::Mat hwc_to_chw(const cv::Mat& src); + }; +} diff --git a/src/MaaWpfGui/Main/AsstProxy.cs b/src/MaaWpfGui/Main/AsstProxy.cs index f1c5723158..953b7b3e13 100644 --- a/src/MaaWpfGui/Main/AsstProxy.cs +++ b/src/MaaWpfGui/Main/AsstProxy.cs @@ -13,6 +13,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; using System.Text; @@ -158,6 +159,7 @@ namespace MaaWpfGui.Main // model references private readonly SettingsViewModel _settingsViewModel; + private readonly TaskQueueViewModel _taskQueueViewModel; private readonly RecruitViewModel _recruitViewModel; private readonly CopilotViewModel _copilotViewModel; @@ -781,6 +783,14 @@ namespace MaaWpfGui.Main break; } } + else if (subTask == "CombatRecordRecognitionTask") + { + string what = details["what"]?.ToString(); + if (!string.IsNullOrEmpty(what)) + { + _copilotViewModel.AddLog(what); + } + } } private void ProcSubTaskCompleted(JObject details) @@ -795,6 +805,10 @@ namespace MaaWpfGui.Main { ProcRecruitCalcMsg(details); } + else if (taskChain == "VideoRecognition") + { + ProcVideoRecMsg(details); + } var subTaskDetails = details["details"]; if (taskChain == "Depot") @@ -1039,9 +1053,11 @@ namespace MaaWpfGui.Main LocalizationHelper.GetString("AlgorithmBadge") + ": " + $"{(int)subTaskDetails["total_badges"]}(+{(int)subTaskDetails["badges"]})" + "\n" + LocalizationHelper.GetString("AlgorithmConstructionPoint") + ": " + $"{(int)subTaskDetails["total_construction_points"]}(+{(int)subTaskDetails["construction_points"]})"); break; + case "ReclamationProcedureStart": _taskQueueViewModel.AddLog(LocalizationHelper.GetString("MissionStart") + $" {(int)subTaskDetails["times"]} " + LocalizationHelper.GetString("UnitTime"), UiLogColor.Info); break; + case "ReclamationSmeltGold": _taskQueueViewModel.AddLog(LocalizationHelper.GetString("AlgorithmDoneSmeltGold") + $" {(int)subTaskDetails["times"]} " + LocalizationHelper.GetString("UnitTime")); break; @@ -1100,6 +1116,27 @@ namespace MaaWpfGui.Main } } + private void ProcVideoRecMsg(JObject details) + { + string what = details["what"].ToString(); + + switch (what) + { + case "Finished": + var filename = details["details"]["filename"].ToString(); + _copilotViewModel.AddLog("Save to: " + filename, UiLogColor.Info); + + // string p = @"C:\tmp\this path contains spaces, and,commas\target.txt"; + string args = string.Format("/e, /select, \"{0}\"", filename); + + ProcessStartInfo info = new ProcessStartInfo(); + info.FileName = "explorer"; + info.Arguments = args; + Process.Start(info); + break; + } + } + public bool AsstSetInstanceOption(InstanceOptionKey key, string value) { return AsstSetInstanceOption(_handle, (AsstInstanceOptionKey)key, value); @@ -1202,6 +1239,7 @@ namespace MaaWpfGui.Main Roguelike, RecruitCalc, Copilot, + VideoRec, Depot, } @@ -1621,6 +1659,17 @@ namespace MaaWpfGui.Main return id != 0 && AsstStart(); } + public bool AsstStartVideoRec(string filename) + { + var task_params = new JObject + { + ["filename"] = filename, + }; + AsstTaskId id = AsstAppendTaskWithEncoding("VideoRecognition", task_params); + _latestTaskId[TaskType.Copilot] = id; + return id != 0 && AsstStart(); + } + /// /// 启动。 /// diff --git a/src/MaaWpfGui/ViewModels/UI/CopilotViewModel.cs b/src/MaaWpfGui/ViewModels/UI/CopilotViewModel.cs index 3f38e3b510..d8011fedc9 100644 --- a/src/MaaWpfGui/ViewModels/UI/CopilotViewModel.cs +++ b/src/MaaWpfGui/ViewModels/UI/CopilotViewModel.cs @@ -172,10 +172,19 @@ namespace MaaWpfGui.ViewModels.UI { ClearLog(); Url = CopilotUiUrl; + _isVideoTask = false; string jsonStr; if (File.Exists(filename)) { + var fileSize = new FileInfo(filename).Length; + bool isJsonFile = filename.ToLower().EndsWith(".json") || fileSize < 4 * 1024 * 1024; + if (!isJsonFile) + { + _isVideoTask = true; + return; + } + try { using var reader = new StreamReader(File.OpenRead(filename)); @@ -393,7 +402,7 @@ namespace MaaWpfGui.ViewModels.UI { var dialog = new OpenFileDialog { - Filter = "JSON|*.json", + Filter = "JSON|*.json|Video|*.mp4;*.m4s;*.mkv;*.flv;*.avi", }; if (dialog.ShowDialog() == true) @@ -402,6 +411,8 @@ namespace MaaWpfGui.ViewModels.UI } } + private static readonly string[] SupportExt = { ".json", ".mp4", ".m4s", ".mkv", ".flv", ".avi" }; + /// /// Drops file. /// @@ -420,7 +431,18 @@ namespace MaaWpfGui.ViewModels.UI return; } - if (filename.EndsWith(".json")) + var filenameLower = filename.ToLower(); + bool support = false; + foreach (var ext in SupportExt) + { + if (filenameLower.EndsWith(ext)) + { + support = true; + break; + } + } + + if (support) { Filename = filename; } @@ -471,6 +493,12 @@ namespace MaaWpfGui.ViewModels.UI }*/ Idle = false; + if (_isVideoTask) + { + StartVideoTask(); + return; + } + AddLog(LocalizationHelper.GetString("ConnectingToEmulator")); string errMsg = string.Empty; @@ -503,6 +531,11 @@ namespace MaaWpfGui.ViewModels.UI } } + public bool StartVideoTask() + { + return _asstProxy.AsstStartVideoRec(Filename); + } + /// /// Stops copilot. /// @@ -512,6 +545,8 @@ namespace MaaWpfGui.ViewModels.UI Idle = true; } + private bool _isVideoTask = false; + private readonly string _copilotRatingUrl = "https://prts.maa.plus/copilot/rating"; private readonly List _recentlyRatedCopilotId = new List(); // TODO: 可能考虑加个持久化