From d2cdb8b2fc04ac1654821f2dec2ca682a4b2a6e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B8=8C=E6=96=AF=E4=BA=9A=E6=89=98=E8=8E=89?= <122478771+HisAtri@users.noreply.github.com> Date: Thu, 13 Jul 2023 19:42:05 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E5=A4=9A=E9=95=9C=E5=83=8F=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E8=AF=B7=E6=B1=82=EF=BC=8C=E9=81=BF=E5=85=8D=E7=BC=93?= =?UTF-8?q?=E6=85=A2=E4=B8=8B=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Python/downloader.py | 86 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 src/Python/downloader.py diff --git a/src/Python/downloader.py b/src/Python/downloader.py new file mode 100644 index 0000000000..41d418310e --- /dev/null +++ b/src/Python/downloader.py @@ -0,0 +1,86 @@ +import requests, os +from concurrent.futures import ThreadPoolExecutor, as_completed +from threading import Lock + +# 获取文件大小 +def length(urlist): + def getlenhead(url): + response = requests.head(url) + file_size = response.headers.get('Content-Length') + if file_size is not None: + return int(file_size) + else: + return False + for url in urlist: + length = getlenhead(url) + if(length): + return length + +# 定义Download类在初始化时保存几个参数 +class Downloader: + # 初始化类 + def __init__(self, urlist, chunksize, max_conn): + self.urlist = urlist # 镜像url列表 + self.chunksize = chunksize # 分片大小 + self.max_conn = max_conn # 单个url最大连接数 + self.lock = Lock() + self.chunk_status = [] # 状态列表 + + def download_chunk(self, url, chunk_id, filepath): + start = chunk_id * self.chunksize + end = start + self.chunksize - 1 + headers = {'Range': f'bytes={start}-{end}'} # 定义header，只定义Range + try: + with self.lock: + if self.chunk_status[chunk_id] != 2: # 检查如果还未下载完（0或1） + self.chunk_status[chunk_id] = 1 # 将当前区块标注为正在下载 + else: + return + response = requests.get(url, headers=headers) + if response.status_code in (301, 302, 303, 307, 308): # 处理HTTP 3xx 重定向问题，继续发送原来的header（range） + redirect_url = response.headers['Location'] + response = requests.get(redirect_url, headers=headers) + if response.status_code == 206: # 206状态码 + with self.lock: + if self.chunk_status[chunk_id] != 2: # 检查是否已经下载完成，如果还没下载完成则写入 + self.write(filepath, response.content, start) + self.chunk_status[chunk_id] = 2 # 将状态标注为已完成 + except requests.RequestException as e: # 处理下载出错的情况 + with self.lock: + if self.chunk_status[chunk_id] == 1: # 如果当前区块状态为正在下载中 + self.chunk_status[chunk_id] = 0 # 重置状态为0，使其它线程优先接手该区块 + + def write(self, filepath, data, position): + with open(filepath, 'rb+') as file: + file.seek(position) + file.write(data) + + def download_file(self, total_size, filepath): + num_chunks = (total_size + self.chunksize - 1) // self.chunksize + self.chunk_status = [0] * num_chunks + with ThreadPoolExecutor(max_workers=self.max_conn * len(self.urlist)) as executor: + futures = {executor.submit(self.download_chunk, url, chunk_id, filepath) + for chunk_id in range(num_chunks) for url in self.urlist} + for future in as_completed(futures): + future.result() + + # 验证下载文件大小 + if os.path.getsize(filepath) != total_size: + print("文件大小不符合，下载可能出错。") + +def file_download(urlist): + chunksize = 1024 * 512 # 分片大小512KB + max_conn = 2 + # 创建对象 + downloader = Downloader(urlist, chunksize, max_conn) + + # 下载文件 + total_size = length(urlist) + filepath = 'path/to/your/file' + return downloader.download_file(total_size, filepath) + +if __name__ == '__main__': + urlist = [] + statu = file_download(urlist) + if statu: + print("下载完成") \ No newline at end of file From cd5bc1210af0c16fcde81bba7f180096e635a737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B8=8C=E6=96=AF=E4=BA=9A=E6=89=98=E8=8E=89?= <122478771+HisAtri@users.noreply.github.com> Date: Thu, 13 Jul 2023 23:28:09 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E5=90=8C=E6=97=B6=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E5=A4=9A=E4=B8=AA=E9=95=9C=E5=83=8Furl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit *不适合有API请求限制的服务器=。=* --- src/Python/downloader.py | 102 ++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 44 deletions(-) diff --git a/src/Python/downloader.py b/src/Python/downloader.py index 41d418310e..4b7b870350 100644 --- a/src/Python/downloader.py +++ b/src/Python/downloader.py @@ -1,6 +1,9 @@ -import requests, os -from concurrent.futures import ThreadPoolExecutor, as_completed +import os +import shutil +import tempfile +from concurrent.futures import ThreadPoolExecutor from threading import Lock +import requests # 获取文件大小 def length(urlist): @@ -19,68 +22,79 @@ def length(urlist): # 定义Download类在初始化时保存几个参数 class Downloader: # 初始化类 - def __init__(self, urlist, chunksize, max_conn): + def __init__(self, urlist, chunksize, max_conn, proxies=None): self.urlist = urlist # 镜像url列表 self.chunksize = chunksize # 分片大小 self.max_conn = max_conn # 单个url最大连接数 + self.proxies = proxies # 代理服务器 self.lock = Lock() self.chunk_status = [] # 状态列表 + self.failed_requests = {url: {'success': 0, 'fail': 0} for url in urlist} # 记录每个 URL 的失败次数和成功次数 - def download_chunk(self, url, chunk_id, filepath): + def download_chunk(self, url, chunk_id, total_size): start = chunk_id * self.chunksize - end = start + self.chunksize - 1 - headers = {'Range': f'bytes={start}-{end}'} # 定义header，只定义Range - try: - with self.lock: - if self.chunk_status[chunk_id] != 2: # 检查如果还未下载完（0或1） - self.chunk_status[chunk_id] = 1 # 将当前区块标注为正在下载 - else: - return - response = requests.get(url, headers=headers) - if response.status_code in (301, 302, 303, 307, 308): # 处理HTTP 3xx 重定向问题，继续发送原来的header（range） - redirect_url = response.headers['Location'] - response = requests.get(redirect_url, headers=headers) - if response.status_code == 206: # 206状态码 - with self.lock: - if self.chunk_status[chunk_id] != 2: # 检查是否已经下载完成，如果还没下载完成则写入 - self.write(filepath, response.content, start) - self.chunk_status[chunk_id] = 2 # 将状态标注为已完成 - except requests.RequestException as e: # 处理下载出错的情况 - with self.lock: - if self.chunk_status[chunk_id] == 1: # 如果当前区块状态为正在下载中 - self.chunk_status[chunk_id] = 0 # 重置状态为0，使其它线程优先接手该区块 + end = min(start + self.chunksize - 1, total_size - 1) + headers = {'Range': f'bytes={start}-{end}'} + filename = f"temp/{chunk_id}" + if self.chunk_status[chunk_id] != 2: + try: + response = requests.get(url, headers=headers, proxies=self.proxies) + if response.status_code in (301, 302, 303, 307, 308): # 处理HTTP 3xx 重定向问题，继续发送原来的header（range） + redirect_url = response.headers['Location'] + response = requests.get(redirect_url, headers=headers, timeout=3, proxies=self.proxies) - def write(self, filepath, data, position): - with open(filepath, 'rb+') as file: - file.seek(position) - file.write(data) + if response.status_code == 206: # 206状态码 + self.failed_requests[url]['success'] += 1 + with open(filename, 'wb') as file: + file.write(response.content) + self.chunk_status[chunk_id] = 2 + elif response.status_code >= 400: + self.failed_requests[url]['fail'] += 1 + + if self.failed_requests[url]['fail'] / (self.failed_requests[url]['success'] + self.failed_requests[url]['fail']) > 0.5 and self.failed_requests[url]['fail'] > 10: + # 如果某 URL 的失败率高于 50%，则跳过该 URL + return + except requests.RequestException as e: + if self.chunk_status[chunk_id] == 1: + self.chunk_status[chunk_id] = 0 def download_file(self, total_size, filepath): num_chunks = (total_size + self.chunksize - 1) // self.chunksize self.chunk_status = [0] * num_chunks + os.makedirs('temp', exist_ok=True) with ThreadPoolExecutor(max_workers=self.max_conn * len(self.urlist)) as executor: - futures = {executor.submit(self.download_chunk, url, chunk_id, filepath) - for chunk_id in range(num_chunks) for url in self.urlist} - for future in as_completed(futures): - future.result() + for url in self.urlist: + for chunk_id in range(num_chunks): + executor.submit(self.download_chunk, url, chunk_id, total_size) - # 验证下载文件大小 + # 合并所有临时文件到一个文件 + with open(filepath, 'wb') as outfile: + for chunk_id in range(num_chunks): + filename = f"temp/{chunk_id}" + with open(filename, 'rb') as infile: + shutil.copyfileobj(infile, outfile) + + # 删除临时目录 + shutil.rmtree('temp') + + # 验证下载文件 if os.path.getsize(filepath) != total_size: - print("文件大小不符合，下载可能出错。") + print("文件大小不一致，下载可能出错。") -def file_download(urlist): - chunksize = 1024 * 512 # 分片大小512KB - max_conn = 2 +def file_download(urlist, filepath, proxies=None): + chunksize = 1024 * 1024 # 分片大小1MB + max_conn = 4 # 创建对象 - downloader = Downloader(urlist, chunksize, max_conn) + downloader = Downloader(urlist, chunksize, max_conn, proxies=proxies) # 下载文件 total_size = length(urlist) - filepath = 'path/to/your/file' + print("文件大小已获取，开始下载") return downloader.download_file(total_size, filepath) if __name__ == '__main__': - urlist = [] - statu = file_download(urlist) - if statu: - print("下载完成") \ No newline at end of file + # 测试。。。 + urlist = [r"https://mirror.eh.cx/KataGo/katago/pack/2023-06-15/2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip",r"https://ota.eh.cx/KataGo/katago/pack/2023-06-15/2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip",r"https://fake.eh.cx/a"] + # 其中前两个是有效的url，第三个是无效的url + path = r"D:\Downloads\2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip" + file_download(urlist,path) From 74a3decb3e089ee2f4e0fef4f4a0082f41b24ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B8=8C=E6=96=AF=E4=BA=9A=E6=89=98=E8=8E=89?= <122478771+HisAtri@users.noreply.github.com> Date: Fri, 14 Jul 2023 09:21:17 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=BB=BA=E7=AB=8B?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E7=9B=AE=E5=BD=95=E7=9A=84=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Python/downloader.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Python/downloader.py b/src/Python/downloader.py index 4b7b870350..8b35064bb0 100644 --- a/src/Python/downloader.py +++ b/src/Python/downloader.py @@ -29,16 +29,17 @@ class Downloader: self.proxies = proxies # 代理服务器 self.lock = Lock() self.chunk_status = [] # 状态列表 - self.failed_requests = {url: {'success': 0, 'fail': 0} for url in urlist} # 记录每个 URL 的失败次数和成功次数 + self.failed_requests = {url: {'success': 0, 'fail': 0} for url in urlist} # 记录每个 URL 的失败次数和成功次数 + self.listhash = hex(hash(tuple(urlist))) # 计算urlist的hash def download_chunk(self, url, chunk_id, total_size): start = chunk_id * self.chunksize end = min(start + self.chunksize - 1, total_size - 1) headers = {'Range': f'bytes={start}-{end}'} - filename = f"temp/{chunk_id}" + filename = f"temp/{self.listhash}/{chunk_id}" if self.chunk_status[chunk_id] != 2: try: - response = requests.get(url, headers=headers, proxies=self.proxies) + response = requests.get(url, headers=headers, proxies=self.proxies, timeout=5) if response.status_code in (301, 302, 303, 307, 308): # 处理HTTP 3xx 重定向问题，继续发送原来的header（range） redirect_url = response.headers['Location'] response = requests.get(redirect_url, headers=headers, timeout=3, proxies=self.proxies) @@ -61,7 +62,11 @@ class Downloader: def download_file(self, total_size, filepath): num_chunks = (total_size + self.chunksize - 1) // self.chunksize self.chunk_status = [0] * num_chunks - os.makedirs('temp', exist_ok=True) + try: + shutil.rmtree(f'temp/{self.listhash}') + except: + pass + os.makedirs(f'temp/{self.listhash}', exist_ok=True) with ThreadPoolExecutor(max_workers=self.max_conn * len(self.urlist)) as executor: for url in self.urlist: for chunk_id in range(num_chunks): @@ -70,12 +75,12 @@ class Downloader: # 合并所有临时文件到一个文件 with open(filepath, 'wb') as outfile: for chunk_id in range(num_chunks): - filename = f"temp/{chunk_id}" + filename = f"temp/{self.listhash}/{chunk_id}" with open(filename, 'rb') as infile: shutil.copyfileobj(infile, outfile) # 删除临时目录 - shutil.rmtree('temp') + shutil.rmtree(f'temp/{self.listhash}') # 验证下载文件 if os.path.getsize(filepath) != total_size: @@ -93,8 +98,7 @@ def file_download(urlist, filepath, proxies=None): return downloader.download_file(total_size, filepath) if __name__ == '__main__': - # 测试。。。 urlist = [r"https://mirror.eh.cx/KataGo/katago/pack/2023-06-15/2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip",r"https://ota.eh.cx/KataGo/katago/pack/2023-06-15/2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip",r"https://fake.eh.cx/a"] # 其中前两个是有效的url，第三个是无效的url - path = r"D:\Downloads\2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip" + path = r"F:\programs\credit\req\2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip" file_download(urlist,path) From 94880559574c636bf64599a9fb6f9d5347984733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B8=8C=E6=96=AF=E4=BA=9A=E6=89=98=E8=8E=89?= <122478771+HisAtri@users.noreply.github.com> Date: Fri, 14 Jul 2023 12:19:21 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20=5F=5Fname=5F=5F=20=3D?= =?UTF-8?q?=3D=20'=5F=5Fmain=5F=5F'=20=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Python/downloader.py | 42 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/Python/downloader.py b/src/Python/downloader.py index 8b35064bb0..66e0912078 100644 --- a/src/Python/downloader.py +++ b/src/Python/downloader.py @@ -1,5 +1,6 @@ import os import shutil +import json import tempfile from concurrent.futures import ThreadPoolExecutor from threading import Lock @@ -98,7 +99,40 @@ def file_download(urlist, filepath, proxies=None): return downloader.download_file(total_size, filepath) if __name__ == '__main__': - urlist = [r"https://mirror.eh.cx/KataGo/katago/pack/2023-06-15/2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip",r"https://ota.eh.cx/KataGo/katago/pack/2023-06-15/2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip",r"https://fake.eh.cx/a"] - # 其中前两个是有效的url，第三个是无效的url - path = r"F:\programs\credit\req\2023-06-15-Macosx(amd64)+Linux(amd64)(无引擎).zip" - file_download(urlist,path) + while True: + json_string = input("请输入下载指令：") + # 解析JSON字符串 + """ + Example: + { + "urlist":["https://mirror1.example.com/file.zip","https://mirror2.example.com/file.zip","https://mirror3.example.com/file.zip"], + "filepath":"C:\Download\File.zip" + "proxies":"http://127.0.0.1:7890" + } + """ + try: + data = json.loads(json_string) + if isinstance(data, dict): + # 这两个参数是必须的 + urlist = data.get("urlist", None) + filepath = data.get("filepath", None) + # 这个参数是可选的 + proxies = data.get("proxies", None) + + # 对于变量的类型进行检查 + if not isinstance(urlist, list) or not all(isinstance(url, str) for url in urlist): + print("urlist类型错误") + continue + if not isinstance(filepath, str): + print("filepath类型错误") + continue + if proxies is not None and not isinstance(proxies, str): + print("proxies类型错误") + continue + + # 启动下载 + file_download(urlist=urlist, filepath=filepath, proxies=proxies) + else: + print("Json结构错误") + except json.JSONDecodeError: + print("无法解析JSON。")