fz-mirrors/MaaAssistantArknights
0
0
Fork 0
mirror of https://github.com/MaaAssistantArknights/MaaAssistantArknights.git synced 2026-07-17 01:59:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: mumu extras (#8939)

Browse Source 
Co-authored-by: uye <99072975+ABA2396@users.noreply.github.com>
Co-authored-by: status102 <102887808+status102@users.noreply.github.com>
This commit is contained in:
MistEO
2024-05-03 22:05:37 +08:00
committed by GitHub
parent 0165469ad0
commit 605dcda98a
31 changed files with 2239 additions and 1175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

323
src/MaaCore/Controller/AdbController.cpp
View File

@@ -8,7 +8,7 @@
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4068)
#pragma warning(disable: 4068)
#endif
#include <zlib/decompress.hpp>
#ifdef _MSC_VER
@@ -18,10 +18,12 @@
#include "Common/AsstTypes.h"
#include "Config/GeneralConfig.h"
#include "Utils/Logger.hpp"
#include "Utils/Platform.hpp"
#include "Utils/StringMisc.hpp"
asst::AdbController::AdbController(const AsstCallback& callback, Assistant* inst, PlatformType type)
: InstHelper(inst), m_callback(callback)
: InstHelper(inst)
, m_callback(callback)
{
LogTraceFunction;
@@ -47,7 +49,8 @@ asst::AdbController::~AdbController()
release();
}
std::optional<std::string> asst::AdbController::reconnect(const std::string& cmd, int64_t timeout, bool recv_by_socket)
std::optional<std::string>
asst::AdbController::reconnect(const std::string& cmd, int64_t timeout, bool recv_by_socket)
{
LogTraceFunction;
@@ -73,7 +76,8 @@ std::optional<std::string> asst::AdbController::reconnect(const std::string& cmd
if (need_exit()) {
break;
}
auto reconnect_ret = call_command(m_adb.connect, 60LL * 1000, false /* 禁止重连避免无限递归 */);
auto reconnect_ret =
call_command(m_adb.connect, 60LL * 1000, false /* 禁止重连避免无限递归 */);
if (need_exit()) {
break;
}
@@ -83,7 +87,8 @@ std::optional<std::string> asst::AdbController::reconnect(const std::string& cmd
is_reconnect_success = reconnect_str.find("error") == std::string::npos;
}
if (is_reconnect_success) {
auto recall_ret = call_command(cmd, timeout, false /* 禁止重连避免无限递归 */, recv_by_socket);
auto recall_ret =
call_command(cmd, timeout, false /* 禁止重连避免无限递归 */, recv_by_socket);
if (recall_ret) {
// 重连并成功执行了
reconnect_info["what"] = "Reconnected";
@@ -108,8 +113,11 @@ std::optional<std::string> asst::AdbController::reconnect(const std::string& cmd
return std::nullopt;
}
std::optional<std::string> asst::AdbController::call_command(const std::string& cmd, int64_t timeout,
bool allow_reconnect, bool recv_by_socket)
std::optional<std::string> asst::AdbController::call_command(
const std::string& cmd,
int64_t timeout,
bool allow_reconnect,
bool recv_by_socket)
{
using namespace std::chrono_literals;
using namespace std::chrono;
@@ -125,7 +133,8 @@ std::optional<std::string> asst::AdbController::call_command(const std::string&
std::optional<int> exit_res;
exit_res = m_platform_io->call_command(cmd, recv_by_socket, pipe_data, sock_data, timeout, start_time);
exit_res =
m_platform_io->call_command(cmd, recv_by_socket, pipe_data, sock_data, timeout, start_time);
if (!exit_res) {
return std::nullopt;
@@ -135,8 +144,17 @@ std::optional<std::string> asst::AdbController::call_command(const std::string&
callcmd_lock.unlock();
m_last_command_duration = duration_cast<milliseconds>(steady_clock::now() - start_time).count();
Log.info("Call `", cmd, "` ret", exit_ret, ", cost", m_last_command_duration, "ms , stdout size:", pipe_data.size(),
", socket size:", sock_data.size());
Log.info(
"Call `",
cmd,
"` ret",
exit_ret,
", cost",
m_last_command_duration,
"ms , stdout size:",
pipe_data.size(),
", socket size:",
sock_data.size());
if (!pipe_data.empty() && pipe_data.size() < 4096) {
Log.trace("stdout output:", Logger::separator::newline, pipe_data);
}
@@ -167,6 +185,19 @@ void asst::AdbController::callback(AsstMsg msg, const json::value& details)
}
}
void asst::AdbController::init_mumu_extras(const AdbCfg& adb_cfg)
{
if (adb_cfg.extras.empty()) {
LogWarn << "adb_cfg.extras is empty";
return;
}
std::filesystem::path mumu_path = utils::path(adb_cfg.extras.get("path", ""));
int mumu_index = adb_cfg.extras.get("index", 0);
int mumu_display = adb_cfg.extras.get("display", 0);
m_mumu_extras.init(mumu_path, mumu_index, mumu_display);
}
void asst::AdbController::close_socket() noexcept
{
m_platform_io->close_socket();
@@ -216,14 +247,20 @@ bool asst::AdbController::click(const Point& p)
Log.error("click point out of range");
}
std::string cur_cmd =
utils::string_replace_all(m_adb.click, { { "[x]", std::to_string(p.x) }, { "[y]", std::to_string(p.y) } });
std::string cur_cmd = utils::string_replace_all(
m_adb.click,
{ { "[x]", std::to_string(p.x) }, { "[y]", std::to_string(p.y) } });
return call_command(cur_cmd).has_value();
}
bool asst::AdbController::swipe(const Point& p1, const Point& p2, int duration, bool extra_swipe,
[[maybe_unused]] double slope_in, [[maybe_unused]] double slope_out,
[[maybe_unused]] bool with_pause)
bool asst::AdbController::swipe(
const Point& p1,
const Point& p2,
int duration,
bool extra_swipe,
[[maybe_unused]] double slope_in,
[[maybe_unused]] double slope_out,
[[maybe_unused]] bool with_pause)
{
int x1 = p1.x, y1 = p1.y;
int x2 = p2.x, y2 = p2.y;
@@ -238,26 +275,31 @@ bool asst::AdbController::swipe(const Point& p1, const Point& p2, int duration,
const auto& opt = Config.get_options();
std::string duration_str =
duration <= 0 ? "" : std::to_string(static_cast<int>(duration * opt.adb_swipe_duration_multiplier));
std::string cur_cmd = utils::string_replace_all(m_adb.swipe, {
{ "[x1]", std::to_string(x1) },
{ "[y1]", std::to_string(y1) },
{ "[x2]", std::to_string(x2) },
{ "[y2]", std::to_string(y2) },
{ "[duration]", duration_str },
});
duration <= 0
? ""
: std::to_string(static_cast<int>(duration * opt.adb_swipe_duration_multiplier));
std::string cur_cmd = utils::string_replace_all(
m_adb.swipe,
{
{ "[x1]", std::to_string(x1) },
{ "[y1]", std::to_string(y1) },
{ "[x2]", std::to_string(x2) },
{ "[y2]", std::to_string(y2) },
{ "[duration]", duration_str },
});
bool ret = call_command(cur_cmd).has_value();
// 额外的滑动：adb有bug，同样的参数，偶尔会划得非常远。额外做一个短程滑动，把之前的停下来
if (extra_swipe && opt.adb_extra_swipe_duration > 0) {
std::string extra_cmd = utils::string_replace_all(
m_adb.swipe, {
{ "[x1]", std::to_string(x2) },
{ "[y1]", std::to_string(y2) },
{ "[x2]", std::to_string(x2) },
{ "[y2]", std::to_string(y2 - opt.adb_extra_swipe_dist /* * m_control_scale*/) },
{ "[duration]", std::to_string(opt.adb_extra_swipe_duration) },
});
m_adb.swipe,
{
{ "[x1]", std::to_string(x2) },
{ "[y1]", std::to_string(y2) },
{ "[x2]", std::to_string(x2) },
{ "[y2]", std::to_string(y2 - opt.adb_extra_swipe_dist /* * m_control_scale*/) },
{ "[duration]", std::to_string(opt.adb_extra_swipe_duration) },
});
ret &= call_command(extra_cmd).has_value();
}
return ret;
@@ -294,7 +336,9 @@ bool asst::AdbController::convert_lf(std::string& data)
if (data.empty() || data.size() < 2) {
return false;
}
auto pred = [](const std::string::iterator& cur) -> bool { return *cur == '\r' && *(cur + 1) == '\n'; };
auto pred = [](const std::string::iterator& cur) -> bool {
return *cur == '\r' && *(cur + 1) == '\n';
};
// find the first of "\r\n"
auto first_iter = data.end();
for (auto iter = data.begin(); iter != data.end() - 1; ++iter) {
@@ -323,23 +367,34 @@ bool asst::AdbController::convert_lf(std::string& data)
bool asst::AdbController::screencap(cv::Mat& image_payload, bool allow_reconnect)
{
using namespace std::chrono;
DecodeFunc decode_raw = [&](const std::string& data) -> bool {
if (data.size() < 8) return false;
if (data.size() < 8) {
return false;
}
// assuming little endian
uint32_t w = static_cast<uint32_t>(static_cast<unsigned char>(data[0])) << 0 |
static_cast<uint32_t>(static_cast<unsigned char>(data[1])) << 8 |
static_cast<uint32_t>(static_cast<unsigned char>(data[2])) << 16 |
static_cast<uint32_t>(static_cast<unsigned char>(data[3])) << 24;
uint32_t h = static_cast<uint32_t>(static_cast<unsigned char>(data[4])) << 0 |
static_cast<uint32_t>(static_cast<unsigned char>(data[5])) << 8 |
static_cast<uint32_t>(static_cast<unsigned char>(data[6])) << 16 |
static_cast<uint32_t>(static_cast<unsigned char>(data[7])) << 24;
uint32_t w = static_cast<uint32_t>(static_cast<unsigned char>(data[0])) << 0
| static_cast<uint32_t>(static_cast<unsigned char>(data[1])) << 8
| static_cast<uint32_t>(static_cast<unsigned char>(data[2])) << 16
| static_cast<uint32_t>(static_cast<unsigned char>(data[3])) << 24;
uint32_t h = static_cast<uint32_t>(static_cast<unsigned char>(data[4])) << 0
| static_cast<uint32_t>(static_cast<unsigned char>(data[5])) << 8
| static_cast<uint32_t>(static_cast<unsigned char>(data[6])) << 16
| static_cast<uint32_t>(static_cast<unsigned char>(data[7])) << 24;
if (int(w) != m_width || int(h) != m_height) {
Log.error("Size from image header", w, h, "does not match the size of screen", m_width, m_height);
Log.error(
"Size from image header",
w,
h,
"does not match the size of screen",
m_width,
m_height);
return false;
}
size_t std_size = 4ULL * m_width * m_height;
if (data.size() < std_size) return false;
if (data.size() < std_size) {
return false;
}
const size_t header_size = data.size() - std_size; // 12 or 16. ref:
// https://android.googlesource.com/platform/frameworks/base/+/26a2b97dbe48ee45e9ae70110714048f2f360f97%5E%21/cmds/screencap/screencap.cpp
auto img_data_beg = data.cbegin() + header_size;
@@ -371,14 +426,13 @@ bool asst::AdbController::screencap(cv::Mat& image_payload, bool allow_reconnect
};
if (m_adb.screencap_method == AdbProperty::ScreencapMethod::UnknownYet) {
using namespace std::chrono;
Log.info("Try to find the fastest way to screencap");
auto min_cost = milliseconds(LLONG_MAX);
clear_lf_info();
auto start_time = high_resolution_clock::now();
if (m_support_socket && m_server_started &&
screencap(m_adb.screencap_raw_by_nc, decode_raw, allow_reconnect, true, 5000)) {
if (m_support_socket && m_server_started
&& screencap(m_adb.screencap_raw_by_nc, decode_raw, allow_reconnect, true, 5000)) {
auto duration = duration_cast<milliseconds>(high_resolution_clock::now() - start_time);
if (duration < min_cost) {
m_adb.screencap_method = AdbProperty::ScreencapMethod::RawByNc;
@@ -420,13 +474,37 @@ bool asst::AdbController::screencap(cv::Mat& image_payload, bool allow_reconnect
else {
Log.info("Encode is not supported");
}
if (m_mumu_extras.inited()) {
start_time = high_resolution_clock::now();
if (m_mumu_extras.screencap()) {
auto duration =
duration_cast<milliseconds>(high_resolution_clock::now() - start_time);
if (duration < min_cost) {
m_adb.screencap_method = AdbProperty::ScreencapMethod::MumuExtras;
m_inited = true;
min_cost = duration;
}
Log.info("MumuExtras cost", duration.count(), "ms");
}
else {
Log.info("MumuExtras is not supported");
}
}
static const std::unordered_map<AdbProperty::ScreencapMethod, std::string> MethodName = {
{ AdbProperty::ScreencapMethod::UnknownYet, "UnknownYet" },
{ AdbProperty::ScreencapMethod::RawByNc, "RawByNc" },
{ AdbProperty::ScreencapMethod::RawWithGzip, "RawWithGzip" },
{ AdbProperty::ScreencapMethod::Encode, "Encode" },
{ AdbProperty::ScreencapMethod::MumuExtras, "MumuExtras" },
};
Log.info("The fastest way is", MethodName.at(m_adb.screencap_method), ", cost:", min_cost.count(), "ms");
Log.info(
"The fastest way is",
MethodName.at(m_adb.screencap_method),
", cost:",
min_cost.count(),
"ms");
if (m_adb.screencap_method != AdbProperty::ScreencapMethod::UnknownYet) {
json::value info = json::object {
{ "uuid", m_uuid },
@@ -443,34 +521,43 @@ bool asst::AdbController::screencap(cv::Mat& image_payload, bool allow_reconnect
return m_adb.screencap_method != AdbProperty::ScreencapMethod::UnknownYet;
}
else {
auto start_time = high_resolution_clock::now();
bool screencap_ret = false;
switch (m_adb.screencap_method) {
case AdbProperty::ScreencapMethod::RawByNc:
screencap_ret = screencap(m_adb.screencap_raw_by_nc, decode_raw, allow_reconnect, true);
break;
case AdbProperty::ScreencapMethod::RawWithGzip:
screencap_ret = screencap(m_adb.screencap_raw_with_gzip, decode_raw_with_gzip, allow_reconnect);
screencap_ret =
screencap(m_adb.screencap_raw_with_gzip, decode_raw_with_gzip, allow_reconnect);
break;
case AdbProperty::ScreencapMethod::Encode:
screencap_ret = screencap(m_adb.screencap_encode, decode_encode, allow_reconnect);
break;
case AdbProperty::ScreencapMethod::MumuExtras: {
auto img_opt = m_mumu_extras.screencap();
screencap_ret = img_opt.has_value();
if (screencap_ret) {
image_payload = img_opt.value();
}
} break;
default:
break;
}
auto duration = duration_cast<milliseconds>(high_resolution_clock::now() - start_time);
// 记录截图耗时，每10次截图回传一次最值+平均值
m_screencap_duration.emplace_back(screencap_ret ? m_last_command_duration : LLONG_MAX); // 记录截图耗时
++m_screencap_time;
m_screencap_cost.emplace_back(screencap_ret ? duration.count() : -1); // 记录截图耗时
++m_screencap_times;
if (m_screencap_duration.size() > 30) {
m_screencap_duration.pop_front();
if (m_screencap_cost.size() > 30) {
m_screencap_cost.pop_front();
}
if (m_screencap_time > 9) { // 每 10 次截图计算一次平均耗时
m_screencap_time = 0;
auto filtered_duration =
m_screencap_duration | views::filter([](long long num) { return num < LLONG_MAX; });
if (m_screencap_times > 9) { // 每 10 次截图计算一次平均耗时
m_screencap_times = 0;
auto filtered_cost = m_screencap_cost | views::filter([](auto num) { return num > 0; });
// 过滤后的有效截图用时次数
auto filtered_count = m_screencap_duration.size() - ranges::count(m_screencap_duration, LLONG_MAX);
auto [screencap_cost_min, screencap_cost_max] = ranges::minmax(filtered_duration);
auto filtered_count = m_screencap_cost.size() - ranges::count(m_screencap_cost, -1);
auto [screencap_cost_min, screencap_cost_max] = ranges::minmax(filtered_cost);
json::value info = json::object {
{ "uuid", m_uuid },
{ "what", "ScreencapCost" },
@@ -480,12 +567,13 @@ bool asst::AdbController::screencap(cv::Mat& image_payload, bool allow_reconnect
{ "max", screencap_cost_max },
{ "avg",
filtered_count > 0
? std::accumulate(filtered_duration.begin(), filtered_duration.end(), 0ll) / filtered_count
? std::accumulate(filtered_cost.begin(), filtered_cost.end(), 0ll)
/ filtered_count
: -1 },
} },
};
if (m_screencap_duration.size() - filtered_count > 0) {
info["details"]["fault_times"] = m_screencap_duration.size() - filtered_count;
if (m_screencap_cost.size() - filtered_count > 0) {
info["details"]["fault_times"] = m_screencap_cost.size() - filtered_count;
}
callback(AsstMsg::ConnectionInfo, info);
}
@@ -493,8 +581,12 @@ bool asst::AdbController::screencap(cv::Mat& image_payload, bool allow_reconnect
}
}
bool asst::AdbController::screencap(const std::string& cmd, const DecodeFunc& decode_func, bool allow_reconnect,
bool by_socket, int timeout)
bool asst::AdbController::screencap(
const std::string& cmd,
const DecodeFunc& decode_func,
bool allow_reconnect,
bool by_socket,
int timeout)
{
if ((!m_support_socket || !m_server_started) && by_socket) [[unlikely]] {
return false;
@@ -517,7 +609,8 @@ bool asst::AdbController::screencap(const std::string& cmd, const DecodeFunc& de
}
if (decode_func(data)) [[likely]] {
if (m_adb.screencap_end_of_line == AdbProperty::ScreencapEndOfLine::UnknownYet) [[unlikely]] {
if (m_adb.screencap_end_of_line == AdbProperty::ScreencapEndOfLine::UnknownYet)
[[unlikely]] {
Log.info("screencap_end_of_line is LF");
m_adb.screencap_end_of_line = AdbProperty::ScreencapEndOfLine::LF;
}
@@ -551,7 +644,10 @@ bool asst::AdbController::screencap(const std::string& cmd, const DecodeFunc& de
return true;
}
bool asst::AdbController::connect(const std::string& adb_path, const std::string& address, const std::string& config)
bool asst::AdbController::connect(
const std::string& adb_path,
const std::string& address,
const std::string& config)
{
LogTraceFunction;
@@ -578,10 +674,11 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
auto adb_ret = Config.get_adb_cfg(config);
if (!adb_ret) {
json::value info = get_info_json() | json::object {
{ "what", "ConnectFailed" },
{ "why", "ConfigNotFound" },
};
json::value info = get_info_json()
| json::object {
{ "what", "ConnectFailed" },
{ "why", "ConfigNotFound" },
};
callback(AsstMsg::ConnectionInfo, info);
return false;
}
@@ -593,13 +690,15 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
// 里面的值每次执行命令后可能更新，所以要用 lambda 拿最新的
auto cmd_replace = [&](const std::string& cfg_cmd) -> std::string {
return utils::string_replace_all(cfg_cmd, {
{ "[Adb]", adb_path },
{ "[AdbSerial]", address },
{ "[DisplayId]", display_id },
{ "[NcPort]", std::to_string(nc_port) },
{ "[NcAddress]", nc_address },
});
return utils::string_replace_all(
cfg_cmd,
{
{ "[Adb]", adb_path },
{ "[AdbSerial]", address },
{ "[DisplayId]", display_id },
{ "[NcPort]", std::to_string(nc_port) },
{ "[NcAddress]", nc_address },
});
};
if (need_exit()) {
@@ -610,19 +709,22 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
{
m_adb.connect = cmd_replace(adb_cfg.connect);
m_adb.release = cmd_replace(adb_cfg.release);
auto connect_ret = call_command(m_adb.connect, 60LL * 1000, false /* adb 连接时不允许重试 */);
auto connect_ret =
call_command(m_adb.connect, 60LL * 1000, false /* adb 连接时不允许重试 */);
bool is_connect_success = false;
if (connect_ret) {
auto& connect_str = connect_ret.value();
is_connect_success = connect_str.find("error") == std::string::npos;
if (connect_str.find("daemon started successfully") != std::string::npos &&
connect_str.find("daemon still not running") == std::string::npos) {}
if (connect_str.find("daemon started successfully") != std::string::npos
&& connect_str.find("daemon still not running") == std::string::npos) {
}
}
if (!is_connect_success) {
json::value info = get_info_json() | json::object {
{ "what", "ConnectFailed" },
{ "why", "Connection command failed to exec" },
};
json::value info = get_info_json()
| json::object {
{ "what", "ConnectFailed" },
{ "why", "Connection command failed to exec" },
};
callback(AsstMsg::ConnectionInfo, info);
return false;
}
@@ -634,12 +736,14 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
/* get uuid (imei) */
{
auto uuid_ret = call_command(cmd_replace(adb_cfg.uuid), 20000, false /* adb 连接时不允许重试 */);
auto uuid_ret =
call_command(cmd_replace(adb_cfg.uuid), 20000, false /* adb 连接时不允许重试 */);
if (!uuid_ret) {
json::value info = get_info_json() | json::object {
{ "what", "ConnectFailed" },
{ "why", "Uuid command failed to exec" },
};
json::value info = get_info_json()
| json::object {
{ "what", "ConnectFailed" },
{ "why", "Uuid command failed to exec" },
};
callback(AsstMsg::ConnectionInfo, info);
return false;
}
@@ -648,10 +752,11 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
std::erase_if(uuid_str, [](char c) { return !std::isdigit(c) && !std::isalpha(c); });
m_uuid = std::move(uuid_str);
json::value info = get_info_json() | json::object {
{ "what", "UuidGot" },
{ "why", "" },
};
json::value info = get_info_json()
| json::object {
{ "what", "UuidGot" },
{ "why", "" },
};
info["details"]["uuid"] = m_uuid;
callback(AsstMsg::ConnectionInfo, info);
}
@@ -687,10 +792,11 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
{
auto display_ret = call_command(cmd_replace(adb_cfg.display));
if (!display_ret) {
json::value info = get_info_json() | json::object {
{ "what", "ConnectFailed" },
{ "why", "Display command failed to exec" },
};
json::value info = get_info_json()
| json::object {
{ "what", "ConnectFailed" },
{ "why", "Display command failed to exec" },
};
callback(AsstMsg::ConnectionInfo, info);
return false;
}
@@ -702,10 +808,11 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
m_width = (std::max)(size_value1, size_value2);
m_height = (std::min)(size_value1, size_value2);
json::value info = get_info_json() | json::object {
{ "what", "ResolutionGot" },
{ "why", "" },
};
json::value info = get_info_json()
| json::object {
{ "what", "ResolutionGot" },
{ "why", "" },
};
info["details"] |= json::object {
{ "width", m_width },
@@ -728,10 +835,11 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
}
{
json::value info = get_info_json() | json::object {
{ "what", "Connected" },
{ "why", "" },
};
json::value info = get_info_json()
| json::object {
{ "what", "Connected" },
{ "why", "" },
};
callback(AsstMsg::ConnectionInfo, info);
}
@@ -777,6 +885,15 @@ bool asst::AdbController::connect(const std::string& adb_path, const std::string
if (need_exit()) {
return false;
}
if (config == "MuMuEmulator12") {
init_mumu_extras(adb_cfg);
}
if (need_exit()) {
return false;
}
return true;
}