From 490cfbf03353d1716ef5728f714f6f31976afc59 Mon Sep 17 00:00:00 2001 From: quyansiyuanwang <132176401+quyansiyuanwang@users.noreply.github.com> Date: Sat, 21 Oct 2023 22:17:34 +0800 Subject: [PATCH] =?UTF-8?q?perf=20&=20refactor:=20=E9=87=8D=E6=9E=84=20too?= =?UTF-8?q?ls/AutoLocalization=20(#6995)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 其中包含部分更优化和跟规范的算法或结构 --------- Co-authored-by: David <56174894+Constrat@users.noreply.github.com> --- tools/AutoLocalization/run.py | 2 +- .../src/auto_localization/cli.py | 140 ++++--- .../src/auto_localization/git.py | 33 +- .../src/auto_localization/translate.py | 233 +++++++----- .../src/auto_localization/xaml_load.py | 342 +++++++++++------- tools/AutoLocalization/tests/__init__.py | 5 + 6 files changed, 474 insertions(+), 281 deletions(-) diff --git a/tools/AutoLocalization/run.py b/tools/AutoLocalization/run.py index 97ca395192..08a3a0dab1 100644 --- a/tools/AutoLocalization/run.py +++ b/tools/AutoLocalization/run.py @@ -1,4 +1,4 @@ from src.auto_localization.cli import main -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/AutoLocalization/src/auto_localization/cli.py b/tools/AutoLocalization/src/auto_localization/cli.py index cc08c53e6b..68a81768a1 100644 --- a/tools/AutoLocalization/src/auto_localization/cli.py +++ b/tools/AutoLocalization/src/auto_localization/cli.py @@ -1,21 +1,40 @@ import argparse import logging import os + from shutil import copy - +from pathlib import Path from dotenv import load_dotenv - from .git import get_latest_file_content from .xaml_load import XamlParser -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logging.debug(os.path.abspath('.env')) -load_dotenv(dotenv_path='.env') -if os.path.exists('../.env'): - load_dotenv(dotenv_path='../.env') -if os.path.exists('../../.env'): - load_dotenv(dotenv_path='../../.env') -root_path = os.getenv("LOCALIZATION_PATH") +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logging.debug(os.path.abspath(".env")) + +solution_dir = Path.cwd() +for i in range(10): + cases = ( + (solution_dir / "docs").exists(), + solution_dir / ".env" in solution_dir.iterdir(), + ) + match cases: + case (_, True): # find .env by find itself + load_dotenv(dotenv_path=solution_dir / ".env") + break + case (True, _): # find .env by finding resource dir + env_path = solution_dir / "tools/AutoLocalization/.env" + if env_path.exists(): + load_dotenv(dotenv_path=env_path) + break + case _: # if not found go up one level + solution_dir = solution_dir.parent +else: + logging.error("未找到.env文件") + exit(1) + +root_path = solution_dir / os.getenv("LOCALIZATION_PATH") assert root_path, "LOCALIZATION_PATH is not set" zh_cn_path = os.path.join(root_path, "zh-cn.xaml") en_us_path = os.path.join(root_path, "en-us.xaml") @@ -26,11 +45,13 @@ ko_kr_path = os.path.join(root_path, "ko-kr.xaml") def initiate(args): logging.info("initiate project") - copy('./copy.env', '.env') - with open('./copy.env', 'r') as f: + copy("./copy.env", ".env") + with open("./copy.env", "r") as f: content = f.read() api_key = input("please input openai api key:") - content = content.replace('OPENAI_API_KEY=YOUR_API_KEY\n', f'OPENAI_API_KEY={api_key}\n') + content = content.replace( + "OPENAI_API_KEY=YOUR_API_KEY\n", f"OPENAI_API_KEY={api_key}\n" + ) with open(".env", "w") as f: f.write(content) @@ -86,7 +107,12 @@ def translate_compare(test): def generate_by_language(test, force, path, language): zh_cn_parser = XamlParser(zh_cn_path) if force: - assert language in ["en-us", "zh-tw", "ja-jp", "ko-kr"], f"language {language} is not supported" + assert language in [ + "en-us", + "zh-tw", + "ja-jp", + "ko-kr", + ], f"language {language} is not supported" language_path = os.path.join(path, f"{language}.xaml") zh_cn_parser.translate_force(language_path, skip_translate=test) return @@ -106,44 +132,61 @@ def generate_by_language(test, force, path, language): def translate_update(test): zh_cn_new_parser = XamlParser(zh_cn_path) - zh_cn_old_content = get_latest_file_content(file_path=zh_cn_path, encoding=zh_cn_new_parser.encoding) - zh_cn_old_parser = XamlParser(parse_type=1, - xaml_string=zh_cn_old_content, - language=zh_cn_new_parser.language, - encoding=zh_cn_new_parser.encoding) + zh_cn_old_content = get_latest_file_content( + file_path=zh_cn_path, encoding=zh_cn_new_parser.encoding + ) + zh_cn_old_parser = XamlParser( + parse_type=1, + xaml_string=zh_cn_old_content, + language=zh_cn_new_parser.language, + encoding=zh_cn_new_parser.encoding, + ) en_us_new_parser = XamlParser(en_us_path) - en_us_old_content = get_latest_file_content(file_path=en_us_path, encoding=en_us_new_parser.encoding) - en_us_old_parser = XamlParser(parse_type=1, - xaml_string=en_us_old_content, - language=en_us_new_parser.language, - encoding=en_us_new_parser.encoding) + en_us_old_content = get_latest_file_content( + file_path=en_us_path, encoding=en_us_new_parser.encoding + ) + en_us_old_parser = XamlParser( + parse_type=1, + xaml_string=en_us_old_content, + language=en_us_new_parser.language, + encoding=en_us_new_parser.encoding, + ) zh_tw_parser = XamlParser(zh_tw_path) ja_jp_parser = XamlParser(ja_jp_path) ko_kr_parser = XamlParser(ko_kr_path) - zh_tw_parser.update_translate(zh_cn_old_parser, zh_cn_new_parser, skip_translate=test) + zh_tw_parser.update_translate( + zh_cn_old_parser, zh_cn_new_parser, skip_translate=test + ) # TODO: 英文更新翻译逻辑? en_us_new_parser.translate_compare(zh_cn_new_parser, skip_translate=test) en_us_new_parser = XamlParser(en_us_path) - ja_jp_parser.update_translate(en_us_old_parser, en_us_new_parser, skip_translate=test) - ko_kr_parser.update_translate(en_us_old_parser, en_us_new_parser, skip_translate=test) + ja_jp_parser.update_translate( + en_us_old_parser, en_us_new_parser, skip_translate=test + ) + ko_kr_parser.update_translate( + en_us_old_parser, en_us_new_parser, skip_translate=test + ) logging.info("update done") def update_by_language(test, path, language): match language: - case s if s in ["en-us", 'zh-tw']: + case s if s in ["en-us", "zh-tw"]: compare_file_path = zh_cn_path case s if s in ["ja-jp", "ko-kr"]: compare_file_path = en_us_path case _: raise ValueError(f"language {language} is not supported") compare_new_parser = XamlParser(compare_file_path) - compare_old_content = get_latest_file_content(file_path=compare_file_path, - encoding=compare_new_parser.encoding) - compare_old_parser = XamlParser(parse_type=1, - xaml_string=compare_old_content, - language=compare_new_parser.language, - encoding=compare_new_parser.encoding) + compare_old_content = get_latest_file_content( + file_path=compare_file_path, encoding=compare_new_parser.encoding + ) + compare_old_parser = XamlParser( + parse_type=1, + xaml_string=compare_old_content, + language=compare_new_parser.language, + encoding=compare_new_parser.encoding, + ) file_path = os.path.join(path, f"{language}.xaml") parser = XamlParser(file_path) parser.update_translate(compare_old_parser, compare_new_parser, skip_translate=test) @@ -152,31 +195,40 @@ def update_by_language(test, path, language): def cli_ui(test=None): parser = argparse.ArgumentParser(description="一个用于自动翻译本地化目录下不同语言文档的命令行工具。") subparsers = parser.add_subparsers() - parser_init = subparsers.add_parser('init', help='初始化工具') + parser_init = subparsers.add_parser("init", help="初始化工具") parser_init.set_defaults(func=initiate) - parser_create = subparsers.add_parser('create', help='初始化其他语言的文档,可选参数：-f --force, -t --test') + parser_create = subparsers.add_parser( + "create", help="初始化其他语言的文档,可选参数：-f --force, -t --test" + ) parser_create.set_defaults(func=create) parser_create.add_argument("-f", "--force", action="store_true", help="强制覆盖已有的部分") - parser_create.add_argument("-t", "--test", action="store_true", help="测试更新情况（跳过chatgpt）") - parser_create.add_argument("-l", "--language", help="指定语言，可选参数：en-us, zh-tw, ja-jp, ko-kr") + parser_create.add_argument( + "-t", "--test", action="store_true", help="测试更新情况（跳过chatgpt）" + ) + parser_create.add_argument( + "-l", "--language", help="指定语言，可选参数：en-us, zh-tw, ja-jp, ko-kr" + ) - parser_update = subparsers.add_parser('update', help='更新本地化翻译,可选参数：-t --test') + parser_update = subparsers.add_parser("update", help="更新本地化翻译,可选参数：-t --test") parser_update.set_defaults(func=update) - parser_update.add_argument("-t", "--test", action="store_true", help="测试更新情况（跳过chatgpt）") - parser_update.add_argument("-l", "--language", help="指定语言，可选参数：en-us, zh-tw, ja-jp, ko-kr") + parser_update.add_argument( + "-t", "--test", action="store_true", help="测试更新情况（跳过chatgpt）" + ) + parser_update.add_argument( + "-l", "--language", help="指定语言，可选参数：en-us, zh-tw, ja-jp, ko-kr" + ) if test: args = parser.parse_args(test) - args.func(args) else: args = parser.parse_args() - args.func(args) + args.func(args) def main(): cli_ui() -if __name__ == '__main__': +if __name__ == "__main__": cli_ui(["init"]) print() diff --git a/tools/AutoLocalization/src/auto_localization/git.py b/tools/AutoLocalization/src/auto_localization/git.py index d9809cb860..6ab49d7b60 100644 --- a/tools/AutoLocalization/src/auto_localization/git.py +++ b/tools/AutoLocalization/src/auto_localization/git.py @@ -1,26 +1,29 @@ import os.path import subprocess - -def get_latest_file_content(file_path='./cli.py', encoding='utf-8', tag_name=""): +def get_latest_file_content(file_path="./cli.py", encoding="utf-8", tag_name=""): dirname = os.path.dirname(file_path) basename = os.path.basename(file_path) # 获取最近一次修改该文件的commit的ID - heading = 'HEAD' + heading = "HEAD" if tag_name: - result = subprocess.run(['git', 'rev-list', '-n', '1', 'tag-autolocalization', '--', basename], - cwd=dirname, - capture_output=True, - text=True) - commit_id = result.stdout.replace('"', '').replace('\'', '').strip() + result = subprocess.run( + ["git", "rev-list", "-n", "1", "tag-autolocalization", "--", basename], + cwd=dirname, + capture_output=True, + text=True, + ) + commit_id = result.stdout.replace('"', "").replace("'", "").strip() heading = commit_id if commit_id else heading - result = subprocess.run(['git', 'show', f'{heading}:./{basename}'], - cwd=dirname, - capture_output=True, - text=True, - encoding=encoding) + result = subprocess.run( + ["git", "show", f"{heading}:./{basename}"], + cwd=dirname, + capture_output=True, + text=True, + encoding=encoding, + ) return result.stdout -if __name__ == '__main__': - print(get_latest_file_content('cli.py')) +if __name__ == "__main__": + print(get_latest_file_content("cli.py")) diff --git a/tools/AutoLocalization/src/auto_localization/translate.py b/tools/AutoLocalization/src/auto_localization/translate.py index 3f842925f6..1301c116bb 100644 --- a/tools/AutoLocalization/src/auto_localization/translate.py +++ b/tools/AutoLocalization/src/auto_localization/translate.py @@ -3,36 +3,55 @@ import logging import os import re import time -from json import JSONDecodeError - import openai + +from pathlib import Path +from json import JSONDecodeError from dotenv import load_dotenv from openai.error import RateLimitError, AuthenticationError from opencc import OpenCC -logging.basicConfig(level=logging.INFO, format='MODULE:%(module)s - %(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig( + level=logging.INFO, + format="MODULE:%(module)s - %(asctime)s - %(levelname)s - %(message)s", +) class ChatTranslator: def __init__(self, language: str = "english", base_language: str = "chinese"): - if os.path.exists('.env'): - load_dotenv(dotenv_path='.env') - elif os.path.exists('../.env'): - load_dotenv(dotenv_path='../.env') - elif os.path.exists('../../.env'): - load_dotenv(dotenv_path='../../.env') + self.base_language = base_language + self.language = language + + solution_dir = Path.cwd() + for i in range(10): + cases = ( + (solution_dir / "docs").exists(), + solution_dir / ".env" in solution_dir.iterdir(), + ) + match cases: + case (_, True): # find .env by find itself + load_dotenv(dotenv_path=solution_dir / ".env") + break + case (True, _): # find .env by finding docs dir + env_path = solution_dir / "tools/AutoLocalization/.env" + if env_path.exists(): + load_dotenv(dotenv_path=env_path) + break + case _: # if not found go up one level + solution_dir = solution_dir.parent else: logging.error("未找到.env文件") exit(1) - self._api_key = os.environ.get('OPENAI_API_KEY') + + self._api_key = os.environ.get("OPENAI_API_KEY") assert self._api_key, "OPENAI_API_KEY is not set" openai.api_key = self._api_key - self._model = os.environ.get('OPENAI_MODEL') + self._model = os.environ.get("OPENAI_MODEL") if not self._model: self._model = "gpt-3.5-turbo" - self._temperature = float(os.environ.get('OPENAI_TEMPERATURE')) + self._temperature = float(os.environ.get("OPENAI_TEMPERATURE")) self._language = language self._base_language = base_language self._rules = """ @@ -59,115 +78,147 @@ class ChatTranslator: @staticmethod def generate_instruction(self, target_language, base_language="Chinese"): - return fr""" + return ( + rf""" 1. Role - you are a translator,don't ask anything, just translate everything i give from {base_language} to {target_language}. - """ + self._rules + """ + """ + + self._rules + + """ - Don't forget the given rules. Now here comes the sentence or words you need to translate, please start to translate:""" + ) - def translate(self, sentence: str = None, target_language: str = None, base_language: str = None, model=None, - temperature: float = None): + def translate( + self, + sentence: str = None, + target_language: str = None, + base_language: str = None, + model=None, + temperature: float = None, + ): # TODO 添加对话长度限制 添加代理 - msg = "" + + if None not in (target_language, base_language): + self.set_language(target_language, base_language) + + if ( + self._language == "Chinese (Traditional)" + and self._base_language == "Chinese (Simplified)" + ): + return OpenCC("s2tw").convert(sentence) # 初始化转换器，s2t表示从简体转繁体 + if sentence is None: sentence = self._test_sentence - if target_language is not None or base_language is not None: - self.set_language(target_language, base_language) if model is None: model = self._model if temperature is None: temperature = self._temperature - if self._language == "Chinese (Traditional)" and self._base_language == "Chinese (Simplified)": - # 初始化转换器，s2t表示从简体转繁体 - cc = OpenCC('s2tw') - return cc.convert(sentence) - new_sentence = sentence.replace(r'\n', r' \n ').replace('\n', ' \n ') - new_sentence = new_sentence.replace(' ', ' ') + + msg = "" + new_sentence = sentence.replace(r"\n", r" \n ").replace("\n", " \n ") + new_sentence = new_sentence.replace(" ", " ") message = [ {"role": "system", "content": self._instruction}, - {"role": "user", "content": '1'}, + {"role": "user", "content": "1"}, {"role": "assistant", "content": '{"message":200,"content":"1"}'}, - {"role": "user", "content": 'abd,l'}, + {"role": "user", "content": "abd,l"}, {"role": "assistant", "content": '{"message":404,"content":"abd,l"}'}, - {"role": "user", "content": '1122 \\n 3344 55666<'}, - {"role": "assistant", "content": '{"message":200,"content":"1122 \\n 3344 55666<"}'}, + {"role": "user", "content": "1122 \\n 3344 55666<"}, + { + "role": "assistant", + "content": '{"message":200,"content":"1122 \\n 3344 55666<"}', + }, {"role": "user", "content": new_sentence}, - ] + + def append_new_msg(i_: int, msg_) -> bool: + if i_ < 9: + message.append({"role": "assistant", "content": msg_}) + message.append({"role": "user", "content": new_sentence}) + return True + return False + + def log_err( + e_=None, info: str = "", append_switch: bool = True + ) -> None: # return it runResult equal to return None + if append_switch: + info = f"{type(e_).__name__}: {e_} msg:{msg}" + info + logging.error(info) + for i in range(10): try: completion = openai.ChatCompletion.create( - model=model, - temperature=temperature, - messages=message + model=model, temperature=temperature, messages=message ) - msg = completion['choices'][0]['message']['content'].strip() - msg = '{' + msg[2:] if msg.startswith("{{") else msg - msg = msg[:-2] + '}' if msg.endswith("}}") else msg - msg = msg.replace('\n', '\\n') if '\n' in msg else msg + msg = completion["choices"][0]["message"]["content"].strip() + + msg = msg[1:] if msg.startswith("{{") else msg + msg = msg[:-1] if msg.endswith("}}") else msg + msg = msg.replace("\n", "\\n") if "\n" in msg else msg + msg_json = json.loads(msg) time.sleep(0.1) - except RateLimitError as _: + + except RateLimitError: time.sleep(2) continue - except JSONDecodeError as _: - if i < 9: - message.append({"role": "assistant", "content": msg}) - message.append({"role": "user", "content": new_sentence}) + + except JSONDecodeError as e: + if append_new_msg(i, msg): continue - pt = re.compile(r"{[^{].*?:.*?,.*?:[^}]*}") - if pt.search(msg): - msg = pt.search(msg).group() + search_msg = re.search(r"{[^{].*?:.*?,.*?:[^}]*}", msg) + if search_msg is not None: + msg = search_msg.group() try: msg_json = json.loads(msg) - except Exception as _: - logging.error(f"{type(_).__name__}: {_} msg:{msg}") - return None + except Exception as e: + return log_err(e) else: - logging.error(f"{type(_).__name__}: {_} msg:{msg}") - return None - except AuthenticationError as _: - logging.error(f"{type(_).__name__}: {_} msg:{msg},maybe key not set correctly") - return None - except Exception as _: - if i < 9: - message.append({"role": "assistant", "content": msg}) - message.append({"role": "user", "content": new_sentence}) - continue - logging.error(f"{type(_).__name__}: {_} msg:{msg}") - return None - match msg_json['message']: - case 200: - # logging.info(f"translate success") - content = msg_json['content'] - # .replace(r'$\\n$', '\\n').replace(r'$\n$', '\\n').replace('$\n$', '\\n') - return content - case 404: - if i < 9: - message.append({"role": "assistant", "content": msg}) - message.append({"role": "user", "content": new_sentence}) - continue - logging.error(f"translate error:{new_sentence}| {msg_json['content']}") - return None - case _: - if i < 9: - message.append({"role": "assistant", "content": msg}) - message.append({"role": "user", "content": new_sentence}) - continue - logging.error(f"translate error: {msg_json}") - return None + return log_err(e) - def set_language(self, target_language, base_language): - self._language = target_language if target_language is not None else self._language - self._base_language = base_language if base_language is not None else self._base_language - self._instruction = self.generate_instruction(self, self._language, self._base_language) + except AuthenticationError as e: + return log_err(e, info=",maybe key not set correctly") + + except Exception as e: + if append_new_msg(i, msg): + continue + return log_err(e) + + if msg_json["message"] == 200: # case 200 + return msg_json["content"] + if append_new_msg(i, msg): + continue + match msg_json["message"]: + case 404: + log_err( + info=f"translate error:{new_sentence}| {msg_json['content']}", + append_switch=False, + ) + + case _: + log_err(info=f"translate error: {msg_json}", append_switch=False) + + def set_language(self, target_language=None, base_language=None): + if target_language is not None: + self._language = target_language + if base_language is not None: + self._base_language = base_language + self._instruction = self.generate_instruction( + self, self._language, self._base_language + ) def add_rules(self, rules: str): - self._rules += """ - - """ + rules.strip() - self._instruction = self.generate_instruction(self, self._language, self._base_language) + self._rules += ( + """ + - """ + + rules.strip() + ) + self._instruction = self.generate_instruction( + self, self._language, self._base_language + ) -# if __name__ == '__main__': -# a = ChatTranslator() -# a.translate() -# print() + +if __name__ == "__main__": + # a = ChatTranslator() + # a.translate() + pass diff --git a/tools/AutoLocalization/src/auto_localization/xaml_load.py b/tools/AutoLocalization/src/auto_localization/xaml_load.py index 76ff19d60a..2d69a65acf 100644 --- a/tools/AutoLocalization/src/auto_localization/xaml_load.py +++ b/tools/AutoLocalization/src/auto_localization/xaml_load.py @@ -1,42 +1,43 @@ import logging import os import re -from copy import deepcopy +from copy import deepcopy from cchardet import detect from lxml import etree from xmldiff import main from xmldiff.actions import UpdateTextIn, InsertComment, UpdateTextAfter, DeleteNode - from .translate import ChatTranslator -SPACE = '{http://www.w3.org/XML/1998/namespace}space' +SPACE = "{http://www.w3.org/XML/1998/namespace}space" -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) def judge_encoding(file): - with open(file, 'rb') as _: + with open(file, "rb") as _: content = _.read() result = detect(content) - return result['encoding'] + return result["encoding"] def parse_lang_str(doc_path): lang_str = os.path.basename(doc_path) match lang_str: - case s if 'zh-cn' in s: + case s if "zh-cn" in s: return "Chinese (Simplified)" - case s if 'zh-tw' in s: + case s if "zh-tw" in s: return "Chinese (Traditional)" - case s if 'en-us' in s: + case s if "en-us" in s: return "English" - case s if 'ja-jp' in s: + case s if "ja-jp" in s: return "Japanese" - case s if 'ko-kr' in s: + case s if "ko-kr" in s: return "Korean" case _: - raise ValueError('wrong language input') + raise ValueError("wrong language input") class XamlParser: @@ -45,7 +46,9 @@ class XamlParser: """ - def __init__(self, file=None, parse_type=0, xaml_string=None, language=None, encoding='utf-8'): + def __init__( + self, file=None, parse_type=0, xaml_string=None, language=None, encoding="utf-8" + ): """ 初始化XamlParser类 :param file: 从文件解析时，传入文件路径 @@ -57,34 +60,59 @@ class XamlParser: self.__test = False self.__counter = 0 if parse_type == 0: - self._file_path, self.__encoding, self.__language, xaml_string = self.__from_file(file) + ( + self._file_path, + self.__encoding, + self.__language, + xaml_string, + ) = self.__from_file(file) elif parse_type == 1: - assert language is not None, 'language must be specified when parse_type is 1' + assert ( + language is not None + ), "language must be specified when parse_type is 1" self._file_path, self.__encoding, self.__language = file, encoding, language else: - raise ValueError('parse_type must be 0 or 1') - xaml_string = xaml_string.replace(' ', '\\n') + raise ValueError("parse_type must be 0 or 1") + xaml_string = xaml_string.replace(" ", "\\n") self.__root = etree.fromstring(xaml_string) # 获取命名空间 self.__nsmap = self.__root.nsmap self.__default_namespace = self.__nsmap.get(None) - self.__x_ns = self.__nsmap.get('x') - self.__x_uid_ns, self.__x_key_ns = f'{{{self.__x_ns}}}Uid', f'{{{self.__x_ns}}}Key' - self.__merged_node = self.__root.find('./ResourceDictionary.MergedDictionaries', namespaces=self.__nsmap) + self.__x_ns = self.__nsmap.get("x") + self.__x_uid_ns, self.__x_key_ns = ( + f"{{{self.__x_ns}}}Uid", + f"{{{self.__x_ns}}}Key", + ) + self.__merged_node = self.__root.find( + "./ResourceDictionary.MergedDictionaries", namespaces=self.__nsmap + ) self.__cp_root = self.copy_node(self.__root, cp_text=True) if self.__merged_node is not None: self.__cp_merged_node = self.copy_node(self.__merged_node, cp_text=True) self.__cp_root.append(self.__cp_merged_node) else: self.__cp_merged_node = None - self.__gen_cp_tree_by_traverse(element=self.__root, current_cp_node=self.__cp_root) + self.__gen_cp_tree_by_traverse( + element=self.__root, current_cp_node=self.__cp_root + ) + + def __getattr__(self, prop): + properties_map = { + "encoding": self.__encoding, + "cp_tree": self.__cp_root, + "tree": self.__root, + "x_uid_ns": self.__x_uid_ns, + "x_key_ns": self.__x_key_ns, + "language": self.__language, + } + return properties_map.get(prop, None) @staticmethod def __from_file(file): - assert os.path.exists(file), f'{file}file not exists' + assert os.path.exists(file), f"{file}file not exists" encoding = judge_encoding(file) language = parse_lang_str(file) - with open(file, 'r', encoding=encoding) as f: + with open(file, "r", encoding=encoding) as f: content = f.read() return file, encoding, language, content @@ -98,7 +126,11 @@ class XamlParser: children = list(parent) index = children.index(child) if index == 0: - if self.__merged_node is not None and parent.tag == f'{{{self.__default_namespace}}}ResourceDictionary': + if ( + self.__merged_node is not None + and parent.tag + == f"{{{self.__default_namespace}}}ResourceDictionary" + ): continue # 注释节点为第一个节点 cp_comment = self.copy_node(child, cp_text=False) @@ -107,17 +139,23 @@ class XamlParser: cp_comment = self.copy_node(child, cp_text=False) prev_node = children[index - 1] prev_cp_node = current_cp_node[index - 1] - assert prev_node.tag == prev_cp_node.tag and prev_node.attrib == prev_cp_node.attrib, "前节点类型不一致" + assert ( + prev_node.tag == prev_cp_node.tag + and prev_node.attrib == prev_cp_node.attrib + ), "前节点类型不一致" current_cp_node.append(cp_comment) continue cp_node = self.copy_node(child) - if child.tag == f'{{{self.__default_namespace}}}ResourceDictionary.MergedDictionaries': + if ( + child.tag + == f"{{{self.__default_namespace}}}ResourceDictionary.MergedDictionaries" + ): # 次根 MergedDictionaries: 遍历后退出循环 current_cp_node = self.__cp_merged_node self.__gen_cp_tree_by_traverse(child, current_cp_node=current_cp_node) break - elif child.tag == f'{{{self.__default_namespace}}}ResourceDictionary': + elif child.tag == f"{{{self.__default_namespace}}}ResourceDictionary": # resource dictionary uid组 cp_node.text = child.text current_cp_node.append(cp_node) @@ -166,7 +204,7 @@ class XamlParser: xaml_data = etree.tostring(tree, pretty_print=True, encoding=self.__encoding) # xaml_data = html.unescape(xaml_data.decode(self.__encoding)) xaml_data = xaml_data.decode(self.__encoding) - with open(file_path, 'w', encoding=self.__encoding) as _: + with open(file_path, "w", encoding=self.__encoding) as _: _.write(xaml_data) return True @@ -178,28 +216,35 @@ class XamlParser: """ this_tree = self.cp_tree compare_tree = compare_parser.cp_tree - assert self.x_key_ns == compare_parser.x_key_ns, f"{self.language} 和 {compare_parser.language}x:Key命名空间不一致" - assert self.x_uid_ns == compare_parser.x_uid_ns, f"{self.language} 和 {compare_parser.language}x:Uid命名空间不一致" - pt = re.compile(r'/\*/\*/\*\[\d*]/comment\(\)\[1]') + assert ( + self.x_key_ns == compare_parser.x_key_ns + ), f"{self.language} 和 {compare_parser.language}x:Key命名空间不一致" + assert ( + self.x_uid_ns == compare_parser.x_uid_ns + ), f"{self.language} 和 {compare_parser.language}x:Uid命名空间不一致" + pt = re.compile(r"/\*/\*/\*\[\d*]/comment\(\)\[1]") uniqueattrs = [self.x_key_ns, self.x_uid_ns] ignored_attrs = [SPACE] - res = main.diff_trees(this_tree, compare_tree, diff_options={ - 'F': 0.1, - 'ratio_mode': 'accurate', - 'uniqueattrs': uniqueattrs, - 'ignored_attrs': ignored_attrs, }) - for i in res: - if type(i).__name__ == 'UpdateTextIn' and pt.search(i.node): - continue + res = main.diff_trees( + this_tree, + compare_tree, + diff_options={ + "F": 0.1, + "ratio_mode": "accurate", + "uniqueattrs": uniqueattrs, + "ignored_attrs": ignored_attrs, + }, + ) + if any((isinstance(i, UpdateTextIn) and pt.search(i.node)) for i in res): return False return True def xpath(self, xpath_str, only_one=True, accept_empty=False): ns = self.nsmap - if None in ns: - del ns[None] - if "" in ns: - del ns[""] + del_arr = (None, "") + for remove_elem in del_arr: + if remove_elem in ns: + del ns[remove_elem] search_result = self.tree.xpath(xpath_str, namespaces=ns) if not len(search_result): assert accept_empty, f"xpath: {xpath_str} 搜索结果为空" @@ -235,14 +280,22 @@ class XamlParser: """ target_language = parse_lang_str(target_path) output_tree = deepcopy(self.cp_tree) - chat = None if skip_translate else ChatTranslator(language=target_language, base_language=self.language) - self.counter(start=True, - test=skip_translate, - messages=f"start force translate {self.language} -> {target_language}") + chat = ( + None + if skip_translate + else ChatTranslator(language=target_language, base_language=self.language) + ) + self.counter( + start=True, + test=skip_translate, + messages=f"start force translate {self.language} -> {target_language}", + ) t = self.__merged_node if self.__merged_node is not None else self.__root - for i in t.findall('.//s:String[@x:Key]', namespaces=self.__nsmap): + for i in t.findall(".//s:String[@x:Key]", namespaces=self.__nsmap): key = i.get(self.__x_key_ns) - node = output_tree.find(f'.//s:String[@x:Key="{key}"]', namespaces=self.__nsmap) + node = output_tree.find( + f'.//s:String[@x:Key="{key}"]', namespaces=self.__nsmap + ) node.text = i.text if skip_translate else chat.translate(i.text) node.text = i.text if node.text is None else node.text self.counter(messages=f"translate {i.text[0:20]}") @@ -258,60 +311,94 @@ class XamlParser: """ target_cp_tree = self.cp_tree base_cp_tree = compare_parser.cp_tree - assert self.nsmap == compare_parser.nsmap, f"{self.language} 和 {compare_parser.language}命名空间不一致" - assert self.x_key_ns == compare_parser.x_key_ns, f"{self.language}和{compare_parser.language}x:Key命名空间不一致" - assert self.x_uid_ns == compare_parser.x_uid_ns, f"{self.language}和{compare_parser.language}x:Uid命名空间不一致" + assert ( + self.nsmap == compare_parser.nsmap + ), f"{self.language} 和 {compare_parser.language}命名空间不一致" + assert ( + self.x_key_ns == compare_parser.x_key_ns + ), f"{self.language}和{compare_parser.language}x:Key命名空间不一致" + assert ( + self.x_uid_ns == compare_parser.x_uid_ns + ), f"{self.language}和{compare_parser.language}x:Uid命名空间不一致" uniqueattrs = [self.x_key_ns, self.x_uid_ns] ignored_attrs = [SPACE] - chat = None if skip_translate else ChatTranslator(language=self.language, base_language=compare_parser.language) - self.counter(start=True, - test=skip_translate, - messages=f"start compare translate {compare_parser.language} -> {self.language}") - res = main.diff_trees(target_cp_tree, base_cp_tree, diff_options={ - 'F': 0.1, - 'ratio_mode': 'accurate', - 'uniqueattrs': uniqueattrs, - 'ignored_attrs': ignored_attrs, }) + chat = ( + None + if skip_translate + else ChatTranslator( + language=self.language, base_language=compare_parser.language + ) + ) + self.counter( + start=True, + test=skip_translate, + messages=f"start compare translate {compare_parser.language} -> {self.language}", + ) + res = main.diff_trees( + target_cp_tree, + base_cp_tree, + diff_options={ + "F": 0.1, + "ratio_mode": "accurate", + "uniqueattrs": uniqueattrs, + "ignored_attrs": ignored_attrs, + }, + ) new_action = [] - logging.info(f"all movements contains {len([i for i in res if type(i).__name__ == 'UpdateTextIn'])} steps") + logging.info( + f"all movements contains {len([i for i in res if isinstance(i, UpdateTextIn)])} steps" + ) for i in res: - if type(i).__name__ == 'MoveNode' and 'comment()' in i.node: + if type(i).__name__ == "MoveNode" and "comment()" in i.node: try: str_node = next(compare_parser.xpath(i.target))[i.position] - parts = i.node.split('/') + parts = i.node.split("/") # 提取父节点的路径 - parent_path = '/'.join(parts[:-1]) - node_position = "{}/comment()[{}]".format(i.target, - len(str_node.xpath('preceding-sibling::comment()')) + 1) + parent_path = "/".join(parts[:-1]) + node_position = "{}/comment()[{}]".format( + i.target, + len(str_node.xpath("preceding-sibling::comment()")) + 1, + ) if parent_path == compare_parser.getpath(str_node.getparent()): # 如果父节点路径相同，说明 new_action.append(i) new_action.append(UpdateTextIn(node_position, str_node.text)) continue - new_action.append(InsertComment(i.target, i.position, str_node.text)) + new_action.append( + InsertComment(i.target, i.position, str_node.text) + ) new_action.append(UpdateTextAfter(node_position, str_node.tail)) new_action.append(DeleteNode(i.node)) except Exception as _: logging.warning(f"{type(_)}: {_}") new_action.append(i) - elif type(i).__name__ == 'UpdateTextIn': - if 'comment()' in i.node: + + elif isinstance(i, UpdateTextIn): + if "comment()" in i.node: continue str_node = next(compare_parser.xpath(i.node)) if str_node.text.strip(): - text = str_node.text if skip_translate else chat.translate(str_node.text) + text = ( + str_node.text + if skip_translate + else chat.translate(str_node.text) + ) text = str_node.text if text is None else text else: text = str_node.text self.counter(messages=f"translate {str_node.text[0:20]}") new_action.append(UpdateTextIn(i.node, text)) - elif type(i).__name__ == 'InsertComment': + elif isinstance(i, InsertComment): str_node = next(compare_parser.xpath(i.target))[i.position] if str_node.tag == etree.Comment: - node_position = "{}/comment()[{}]".format(i.target, - len(str_node.xpath('preceding-sibling::comment()')) + 1) - new_action.append(InsertComment(i.target, i.position, str_node.text)) + node_position = "{}/comment()[{}]".format( + i.target, + len(str_node.xpath("preceding-sibling::comment()")) + 1, + ) + new_action.append( + InsertComment(i.target, i.position, str_node.text) + ) new_action.append(UpdateTextAfter(node_position, str_node.tail)) else: new_action.append(i) @@ -322,7 +409,9 @@ class XamlParser: self.write_xaml(final_tree, self._file_path) return final_tree - def update_translate(self, compare_old_parser, compare_new_parser, skip_translate=False): + def update_translate( + self, compare_old_parser, compare_new_parser, skip_translate=False + ): """ 将比较新旧两个文件的更新，并将更新同步到本文件，若本文件与旧文件结构不同，则会与旧文件调用translate_compare :param compare_old_parser: 旧文件的XamlParser @@ -332,75 +421,70 @@ class XamlParser: """ compare_old_tree = compare_old_parser.merged_root_tree if not self.compare_structure(compare_old_parser): - target_tree = self.translate_compare(compare_old_parser, skip_translate=skip_translate) + target_tree = self.translate_compare( + compare_old_parser, skip_translate=skip_translate + ) else: target_tree = self.cp_tree compare_new_tree = compare_new_parser.merged_root_tree - assert compare_old_parser.nsmap == compare_new_parser.nsmap, \ - f"old {compare_old_parser.language}和 new {compare_new_parser.language}命名空间不一致" - assert compare_old_parser.x_key_ns == compare_new_parser.x_key_ns, \ - f"old {compare_old_parser.language}和 new {compare_new_parser.language}x:Key命名空间不一致" - assert compare_old_parser.x_uid_ns == compare_new_parser.x_uid_ns, \ - f"old {compare_old_parser.language}和 new {compare_new_parser.language}x:Uid命名空间不一致" + namespace_ununit_string = "old %s和 new %s{}命名空间不一致" % ( + compare_old_parser.language, + compare_new_parser.language, + ) + for prop, err in {"nsmap": "", "x_key_ns": "x:Key", "x_uid_ns": "x:Uid"}: + assert getattr(compare_old_parser, prop) == getattr( + compare_new_parser, prop + ), namespace_ununit_string.format(err) - chat = None if skip_translate else ChatTranslator(language=self.language, - base_language=compare_new_parser.language) - self.counter(start=True, - test=skip_translate, - messages=f"start compare translate {compare_new_parser.language} -> {self.language}") + chat = ( + None + if skip_translate + else ChatTranslator( + language=self.language, base_language=compare_new_parser.language + ) + ) + self.counter( + start=True, + test=skip_translate, + messages=f"start compare translate {compare_new_parser.language} -> {self.language}", + ) uniqueattrs = [self.x_key_ns, self.x_uid_ns] ignored_attrs = [SPACE] - res = main.diff_trees(compare_old_tree, compare_new_tree, diff_options={ - 'F': 0.1, - 'ratio_mode': 'accurate', - 'uniqueattrs': uniqueattrs, - 'ignored_attrs': ignored_attrs - }) + res = main.diff_trees( + compare_old_tree, + compare_new_tree, + diff_options={ + "F": 0.1, + "ratio_mode": "accurate", + "uniqueattrs": uniqueattrs, + "ignored_attrs": ignored_attrs, + }, + ) new_actions = [] - logging.info(f"all movements contains {len([i for i in res if type(i).__name__ == 'UpdateTextIn'])} steps") + logging.info( + f"all movements contains {len([i for i in res if isinstance(i, UpdateTextIn)])} steps" + ) for i in res: - if type(i).__name__ == 'UpdateTextIn': - if i.node.endswith('comment()[1]') and i.text[0] == '$': + if isinstance(i, UpdateTextIn): + if i.node.endswith("comment()[1]") and i.text[0] == "$": continue - elif 's:String' in i.node: + elif "s:String" in i.node: text = i.text if skip_translate else chat.translate(i.text) text = i.text if text is None else text self.counter(messages=f"translate {i.text[0:20]}") new_actions.append(UpdateTextIn(i.node, text)) - else: - new_actions.append(i) - else: - new_actions.append(i) + continue + new_actions.append(i) final_tree = main.patch_tree(new_actions, target_tree) compare_new_parser.write_xaml() self.write_xaml(final_tree) return final_tree - @property - def encoding(self): - return self.__encoding - - @property - def cp_tree(self): - return self.__cp_root - - @property - def tree(self): - return self.__root - @property def nsmap(self): return deepcopy(self.__nsmap) - @property - def x_uid_ns(self): - return self.__x_uid_ns - - @property - def x_key_ns(self): - return self.__x_key_ns - @property def merged_root_tree(self): root = self.copy_node(self.__root, cp_text=True) @@ -410,16 +494,14 @@ class XamlParser: root = deepcopy(self.__root) return root - @property - def language(self): - return self.__language - @property def tostring(self): - return etree.tostring(self.tree, encoding=self.__encoding, pretty_print=True).decode(self.__encoding) + return etree.tostring( + self.tree, encoding=self.__encoding, pretty_print=True + ).decode(self.__encoding) -if __name__ == '__main__': - zh_parser = XamlParser(parse_type=0, file='../../example/zh-cn.xaml') - zh_new_parser = XamlParser(parse_type=0, file='../../example/zh-cn_new.xaml') +if __name__ == "__main__": + zh_parser = XamlParser(parse_type=0, file="../../example/zh-cn.xaml") + zh_new_parser = XamlParser(parse_type=0, file="../../example/zh-cn_new.xaml") print() diff --git a/tools/AutoLocalization/tests/__init__.py b/tools/AutoLocalization/tests/__init__.py index e69de29bb2..b0447af6f7 100644 --- a/tools/AutoLocalization/tests/__init__.py +++ b/tools/AutoLocalization/tests/__init__.py @@ -0,0 +1,5 @@ +from _test_all import TestAll +from test_translate import TestChatTranslator +from test_xaml_load import TestXamlParser + +__all__ = ['TestAll', 'TestChatTranslator', 'TestXamlParser']