From d970c22e8fa9bbc7459341ae8de6d0547dc96dec Mon Sep 17 00:00:00 2001 From: Constrat <56174894+Constrat@users.noreply.github.com> Date: Tue, 24 Sep 2024 20:20:50 +0200 Subject: [PATCH] feat(perf): multithreaded resource updater (#10683) * feat(perf): multithreading for resource updater * perf: multithread penguin request * style: formatting * test: remove test comments --- tools/ResourceUpdater/main.cpp | 229 +++++++++++++-------------------- 1 file changed, 92 insertions(+), 137 deletions(-) diff --git a/tools/ResourceUpdater/main.cpp b/tools/ResourceUpdater/main.cpp index a4917c8e2d..0353a934a2 100644 --- a/tools/ResourceUpdater/main.cpp +++ b/tools/ResourceUpdater/main.cpp @@ -1,5 +1,8 @@ +#include #include #include +#include +#include #include #include @@ -72,12 +75,19 @@ bool check_roguelike_replace_for_overseas( const std::filesystem::path& output_dir); bool update_version_info(const std::filesystem::path& input_dir, const std::filesystem::path& output_dir); +template +auto run_async(Func&& func, Args&&... args) +{ + return std::async(std::launch::async, std::forward(func), std::forward(args)...); +} + int main([[maybe_unused]] int argc, char** argv) { // ---- PATH DECLARATION ---- const char* str_exec_path = argv[0]; const auto cur_path = std::filesystem::path(str_exec_path).parent_path(); + std::vector> futures; auto solution_dir = cur_path; for (int i = 0; i != 10; ++i) { @@ -102,159 +112,88 @@ int main([[maybe_unused]] int argc, char** argv) // ---- METHODS CALLS ---- - // Update levels.json from ArknightsGameResource - std::cout << "------- Update levels.json for Official -------" << '\n'; - if (!update_levels_json(official_data_dir / "levels.json", resource_dir / "Arknights-Tile-Pos")) { - std::cerr << "update levels.json failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; - } + auto start = std::chrono::high_resolution_clock::now(); - // 这个 en_levels.json 是自己手动生成放进去的 - // Will never work without en_levels.json in proj_dir, commented for now - // generate_english_roguelike_stage_name_replacement(official_data_dir / "levels.json", cur_path - // / "en_levels.json"); + // Update stage.json from Penguin Stats + futures.push_back(run_async(update_stages_data, cur_path, resource_dir)); // Update infrast data from ArknightsGameResource - std::cout << "------- Update infrast data for Official -------" << '\n'; - if (!update_infrast_data(official_data_dir / "gamedata" / "excel", resource_dir)) { - std::cerr << "Update infrast data failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; - } + futures.push_back(run_async(update_infrast_data, official_data_dir / "gamedata" / "excel", resource_dir)); // Update infrast templates from ArknightsGameResource - std::cout << "------- Update infrast templates for Official -------" << '\n'; - if (!update_infrast_templates(official_data_dir / "building_skill", resource_dir / "template" / "infrast")) { - std::cerr << "Update infrast templates failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; - } - - // Update roguelike recruit data from ArknightsGameResource - // std::cout << "------- Update roguelike recruit data -------" << '\n'; - // if (!update_roguelike_recruit(arkbot_res_dir, resource_dir, solution_dir)) { - // std::cerr << "Update roguelike recruit data failed" << '\n'; - // return -1; - // } else { - // std::cout << "Done" << '\n'; - //} - - // Update base_name.json from Penguin Stats - std::cout << "------- Update stage.json for Official -------" << '\n'; - if (!update_stages_data(cur_path, resource_dir)) { - std::cerr << "Update stages data failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; - } + futures.push_back(run_async( + update_infrast_templates, + official_data_dir / "building_skill", + resource_dir / "template" / "infrast")); // Update battle chars info for all clients - std::cout << "------- Update battle chars info for all clients -------" << '\n'; - if (!update_battle_chars_info(official_data_dir / "gamedata" / "excel", overseas_data_dir, resource_dir)) { - std::cerr << "Update battle chars info failed" << '\n'; - return -1; + futures.push_back( + run_async(update_battle_chars_info, official_data_dir / "gamedata" / "excel", overseas_data_dir, resource_dir)); + + // Update roguelike replace for overseas from ArknightsGameData_YoStar + for (const auto& [in, out] : global_dirs) { + futures.push_back(run_async( + check_roguelike_replace_for_overseas, + overseas_data_dir / in / "gamedata" / "excel", + resource_dir / "global" / out / "resource" / "tasks.json", + official_data_dir / "gamedata" / "excel", + cur_path / in)); } - else { - std::cout << "Done" << '\n'; + + // Update version info from ArknightsGameData + futures.push_back(run_async(update_version_info, official_data_dir / "gamedata" / "excel", resource_dir)); + + // Update global version info from ArknightsGameData_YoStar + for (const auto& [in, out] : global_dirs) { + futures.push_back(run_async( + update_version_info, + overseas_data_dir / in / "gamedata" / "excel", + resource_dir / "global" / out / "resource")); } // Update recruitment data from ArknightsGameResource - std::cout << "------- Update recruitment data for Official -------" << '\n'; - if (!update_recruitment_data(official_data_dir / "gamedata" / "excel", resource_dir / "recruitment.json", true)) { - std::cerr << "Update recruitment data failed" << '\n'; + std::future a1 = run_async( + update_recruitment_data, + official_data_dir / "gamedata" / "excel", + resource_dir / "recruitment.json", + true); + + // Update items template and json from ArknightsGameResource + std::future a2 = run_async(update_items_data, official_data_dir, resource_dir, true); + + if (!a1.get() || !a2.get()) { + std::cerr << "One of the dependent tasks in group A failed" << '\n'; return -1; } - else { - std::cout << "Done" << '\n'; - } // Update recruitment data from ArknightsGameData_YoStar for (const auto& [in, out] : global_dirs) { - std::cout << "------- Update recruitment data for " << out << "------- " << '\n'; - if (!update_recruitment_data( - overseas_data_dir / in / "gamedata" / "excel", - resource_dir / "global" / out / "resource" / "recruitment.json", - false)) { - std::cerr << "Update recruitment data failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; - } - } - - // Update items template and json from ArknightsGameResource - std::cout << "------- Update items template and json for Official -------" << '\n'; - if (!update_items_data(official_data_dir, resource_dir)) { - std::cerr << "Update items data failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; + futures.push_back(run_async( + update_recruitment_data, + overseas_data_dir / in / "gamedata" / "excel", + resource_dir / "global" / out / "resource" / "recruitment.json", + false)); } // Update items template and json from ArknightsGameData_YoStar for (const auto& [in, out] : global_dirs) { - std::cout << "------- Update items template and json for " << out << "------- " << '\n'; - if (!update_items_data( - overseas_data_dir / in / "gamedata" / "excel", - resource_dir / "global" / out / "resource", - false)) { - std::cerr << "Update items json failed" << '\n'; + futures.push_back(run_async( + update_items_data, + overseas_data_dir / in / "gamedata" / "excel", + resource_dir / "global" / out / "resource", + false)); + } + + for (auto& future : futures) { + if (!future.get()) { + std::cerr << "One of the updates failed" << '\n'; return -1; } - else { - std::cout << "Done" << '\n'; - } } - // Update roguelike replace for overseas from ArknightsGameData_YoStar - for (const auto& [in, out] : global_dirs) { - std::cout << "------- Update roguelike replace for " << out << "------- " << '\n'; - if (!check_roguelike_replace_for_overseas( - overseas_data_dir / in / "gamedata" / "excel", - resource_dir / "global" / out / "resource" / "tasks.json", - official_data_dir / "gamedata" / "excel", - cur_path / in)) { - std::cerr << "Update roguelike replace for overseas failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; - } - } - - // Update version info from ArknightsGameData - std::cout << "------- Update version info for Official -------" << '\n'; - if (!update_version_info(official_data_dir / "gamedata" / "excel", resource_dir)) { - std::cerr << "Update version info failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; - } - - // Update global version info from ArknightsGameData_YoStar - for (const auto& [in, out] : global_dirs) { - std::cout << "------- Update version info for " << out << "------- " << '\n'; - if (!update_version_info( - overseas_data_dir / in / "gamedata" / "excel", - resource_dir / "global" / out / "resource")) { - std::cerr << "Update version info failed" << '\n'; - return -1; - } - else { - std::cout << "Done" << '\n'; - } - } + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed_seconds = end - start; + std::cout << "Time: " << elapsed_seconds.count() << "s" << '\n'; std::cout << "------- All success -------" << '\n'; return 0; @@ -266,7 +205,6 @@ bool update_items_data(const std::filesystem::path& input_dir, const std::filesy { const auto input_json_path = with_imgs ? input_dir / "gamedata" / "excel" / "item_table.json" : input_dir / "item_table.json"; - auto parse_ret = json::open(input_json_path); if (!parse_ret) { std::cerr << "parse json failed" << '\n'; @@ -562,8 +500,6 @@ bool update_stages_data(const std::filesystem::path& input_dir, const std::files const std::string PenguinAPI = R"(https://penguin-stats.io/PenguinStats/api/v2/stages?server=)"; const std::vector PenguinServers = { "CN", "US", "JP", "KR" }; - const std::filesystem::path TempFile = input_dir / "stages.json"; - struct DropInfo { std::string item_id; @@ -576,16 +512,35 @@ bool update_stages_data(const std::filesystem::path& input_dir, const std::files std::map> drop_infos; std::map stage_basic_infos; - for (const auto& server : PenguinServers) { - int stage_request_ret = system(("curl -o \"" + TempFile.string() + "\" " + PenguinAPI + server).c_str()); + + auto fetch_and_parse = [&](const std::string& server) -> bool { + std::filesystem::path temp_file = input_dir / ("stages_" + server + ".json"); + int stage_request_ret = system(("curl -o \"" + temp_file.string() + "\" " + PenguinAPI + server).c_str()); if (stage_request_ret != 0) { - std::cerr << "Request Penguin Stats failed" << '\n'; + std::cerr << "Request Penguin Stats failed for server: " << server << '\n'; return false; } + return true; + }; - auto parse_ret = json::open(TempFile); + std::vector> futures; + for (const auto& server : PenguinServers) { + futures.push_back(std::async(std::launch::async, fetch_and_parse, server)); + } + + for (auto& future : futures) { + if (!future.get()) { + std::cerr << "One of the updates failed" << '\n'; + return false; + } + } + + // Merge data from all temporary files sequentially + for (const auto& server : PenguinServers) { + std::filesystem::path temp_file = input_dir / ("stages_" + server + ".json"); + auto parse_ret = json::open(temp_file); if (!parse_ret) { - std::cerr << "parse stages.json failed" << '\n'; + std::cerr << "parse stages.json failed for server: " << server << '\n'; return false; } auto& stage_json = parse_ret.value();