mirror of
https://github.com/AstrBotDevs/AstrBot
synced 2026-07-18 10:00:40 +08:00
9afe5757be
* docs: align deployment sections across multilingual readmes * docs: normalize deployment punctuation and AUR guidance * docs: fix french and russian deployment wording * perf: optimize async io hot paths and extend benchmarks * fix: address async io review feedback * fix: address follow-up async io review comments * fix: align base64 io error handling in message components * fix: harden attachment export ids and tune io chunking * fix: preserve best-effort attachment export and batch writes * test: expand path conversion and helper coverage
322 lines
12 KiB
Python
322 lines
12 KiB
Python
import asyncio
|
||
import base64
|
||
import logging
|
||
import os
|
||
import shutil
|
||
import socket
|
||
import ssl
|
||
import time
|
||
import uuid
|
||
import zipfile
|
||
from pathlib import Path
|
||
from typing import BinaryIO
|
||
|
||
import aiohttp
|
||
import certifi
|
||
import psutil
|
||
from PIL import Image
|
||
|
||
from .astrbot_path import get_astrbot_data_path, get_astrbot_path, get_astrbot_temp_path
|
||
|
||
logger = logging.getLogger("astrbot")
|
||
_DOWNLOAD_READ_CHUNK_SIZE = 64 * 1024
|
||
_DOWNLOAD_FLUSH_THRESHOLD = 256 * 1024
|
||
|
||
|
||
def on_error(func, path, exc_info) -> None:
|
||
"""A callback of the rmtree function."""
|
||
import stat
|
||
|
||
if not os.access(path, os.W_OK):
|
||
os.chmod(path, stat.S_IWUSR)
|
||
func(path)
|
||
else:
|
||
raise exc_info[1]
|
||
|
||
|
||
def remove_dir(file_path: str) -> bool:
|
||
if not os.path.exists(file_path):
|
||
return True
|
||
shutil.rmtree(file_path, onerror=on_error)
|
||
return True
|
||
|
||
|
||
def port_checker(port: int, host: str = "localhost") -> bool:
|
||
sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||
sk.settimeout(1)
|
||
try:
|
||
sk.connect((host, port))
|
||
sk.close()
|
||
return True
|
||
except Exception:
|
||
sk.close()
|
||
return False
|
||
|
||
|
||
def save_temp_img(img: Image.Image | bytes) -> str:
|
||
temp_dir = get_astrbot_temp_path()
|
||
# 获得时间戳
|
||
timestamp = f"{int(time.time())}_{uuid.uuid4().hex[:8]}"
|
||
p = os.path.join(temp_dir, f"io_temp_img_{timestamp}.jpg")
|
||
|
||
if isinstance(img, Image.Image):
|
||
img.save(p)
|
||
else:
|
||
Path(p).write_bytes(img)
|
||
return p
|
||
|
||
|
||
async def download_image_by_url(
|
||
url: str,
|
||
post: bool = False,
|
||
post_data: dict | None = None,
|
||
path: str | None = None,
|
||
) -> str:
|
||
"""下载图片, 返回 path"""
|
||
try:
|
||
ssl_context = ssl.create_default_context(
|
||
cafile=certifi.where(),
|
||
) # 使用 certifi 提供的 CA 证书
|
||
connector = aiohttp.TCPConnector(ssl=ssl_context) # 使用 certifi 的根证书
|
||
async with aiohttp.ClientSession(
|
||
trust_env=True,
|
||
connector=connector,
|
||
) as session:
|
||
if post:
|
||
async with session.post(url, json=post_data) as resp:
|
||
if not path:
|
||
return save_temp_img(await resp.read())
|
||
await asyncio.to_thread(Path(path).write_bytes, await resp.read())
|
||
return path
|
||
else:
|
||
async with session.get(url) as resp:
|
||
if not path:
|
||
return save_temp_img(await resp.read())
|
||
await asyncio.to_thread(Path(path).write_bytes, await resp.read())
|
||
return path
|
||
except (aiohttp.ClientConnectorSSLError, aiohttp.ClientConnectorCertificateError):
|
||
# 关闭SSL验证（仅在证书验证失败时作为fallback）
|
||
logger.warning(
|
||
f"SSL certificate verification failed for {url}. "
|
||
"Disabling SSL verification (CERT_NONE) as a fallback. "
|
||
"This is insecure and exposes the application to man-in-the-middle attacks. "
|
||
"Please investigate and resolve certificate issues."
|
||
)
|
||
ssl_context = ssl.create_default_context()
|
||
ssl_context.check_hostname = False
|
||
ssl_context.verify_mode = ssl.CERT_NONE
|
||
async with aiohttp.ClientSession() as session:
|
||
if post:
|
||
async with session.post(url, json=post_data, ssl=ssl_context) as resp:
|
||
if not path:
|
||
return save_temp_img(await resp.read())
|
||
await asyncio.to_thread(Path(path).write_bytes, await resp.read())
|
||
return path
|
||
else:
|
||
async with session.get(url, ssl=ssl_context) as resp:
|
||
if not path:
|
||
return save_temp_img(await resp.read())
|
||
await asyncio.to_thread(Path(path).write_bytes, await resp.read())
|
||
return path
|
||
except Exception as e:
|
||
raise e
|
||
|
||
|
||
async def download_file(url: str, path: str, show_progress: bool = False) -> None:
|
||
"""从指定 url 下载文件到指定路径 path"""
|
||
try:
|
||
ssl_context = ssl.create_default_context(
|
||
cafile=certifi.where(),
|
||
) # 使用 certifi 提供的 CA 证书
|
||
connector = aiohttp.TCPConnector(ssl=ssl_context)
|
||
async with aiohttp.ClientSession(
|
||
trust_env=True,
|
||
connector=connector,
|
||
) as session:
|
||
async with session.get(url, timeout=1800) as resp:
|
||
if resp.status != 200:
|
||
raise Exception(f"下载文件失败: {resp.status}")
|
||
total_size = int(resp.headers.get("content-length", 0))
|
||
start_time = time.time()
|
||
if show_progress:
|
||
print(f"文件大小: {total_size / 1024:.2f} KB | 文件地址: {url}")
|
||
file_obj = await asyncio.to_thread(Path(path).open, "wb")
|
||
try:
|
||
await _stream_to_file(
|
||
resp.content,
|
||
file_obj,
|
||
total_size=total_size,
|
||
start_time=start_time,
|
||
show_progress=show_progress,
|
||
)
|
||
finally:
|
||
await asyncio.to_thread(file_obj.close)
|
||
except (aiohttp.ClientConnectorSSLError, aiohttp.ClientConnectorCertificateError):
|
||
# 关闭SSL验证（仅在证书验证失败时作为fallback）
|
||
logger.warning(
|
||
"SSL 证书验证失败，已关闭 SSL 验证（不安全，仅用于临时下载）。请检查目标服务器的证书配置。"
|
||
)
|
||
logger.warning(
|
||
f"SSL certificate verification failed for {url}. "
|
||
"Falling back to unverified connection (CERT_NONE). "
|
||
"This is insecure and exposes the application to man-in-the-middle attacks. "
|
||
"Please investigate certificate issues with the remote server."
|
||
)
|
||
ssl_context = ssl.create_default_context()
|
||
ssl_context.check_hostname = False
|
||
ssl_context.verify_mode = ssl.CERT_NONE
|
||
async with aiohttp.ClientSession() as session:
|
||
async with session.get(url, ssl=ssl_context, timeout=120) as resp:
|
||
total_size = int(resp.headers.get("content-length", 0))
|
||
start_time = time.time()
|
||
if show_progress:
|
||
print(f"文件大小: {total_size / 1024:.2f} KB | 文件地址: {url}")
|
||
file_obj = await asyncio.to_thread(Path(path).open, "wb")
|
||
try:
|
||
await _stream_to_file(
|
||
resp.content,
|
||
file_obj,
|
||
total_size=total_size,
|
||
start_time=start_time,
|
||
show_progress=show_progress,
|
||
)
|
||
finally:
|
||
await asyncio.to_thread(file_obj.close)
|
||
if show_progress:
|
||
print()
|
||
|
||
|
||
async def _stream_to_file(
|
||
stream: aiohttp.StreamReader,
|
||
file_obj: BinaryIO,
|
||
*,
|
||
total_size: int,
|
||
start_time: float,
|
||
show_progress: bool,
|
||
) -> None:
|
||
"""Stream HTTP response into file with buffered thread-offloaded writes."""
|
||
downloaded_size = 0
|
||
known_total = total_size if total_size > 0 else None
|
||
buffered = bytearray()
|
||
|
||
try:
|
||
while True:
|
||
chunk = await stream.read(_DOWNLOAD_READ_CHUNK_SIZE)
|
||
if not chunk:
|
||
break
|
||
|
||
buffered.extend(chunk)
|
||
downloaded_size += len(chunk)
|
||
|
||
if len(buffered) >= _DOWNLOAD_FLUSH_THRESHOLD:
|
||
await asyncio.to_thread(file_obj.write, bytes(buffered))
|
||
buffered.clear()
|
||
|
||
if show_progress:
|
||
_print_download_progress(downloaded_size, known_total, start_time)
|
||
finally:
|
||
if buffered:
|
||
# Ensure buffered data is flushed even on cancellation.
|
||
await asyncio.shield(asyncio.to_thread(file_obj.write, bytes(buffered)))
|
||
|
||
|
||
def _print_download_progress(
|
||
downloaded_size: int, total_size: int | None, start_time: float
|
||
) -> None:
|
||
elapsed_time = max(time.time() - start_time, 1e-6)
|
||
speed = downloaded_size / 1024 / elapsed_time # KB/s
|
||
|
||
if total_size:
|
||
percent = downloaded_size / total_size
|
||
msg = f"\r下载进度: {percent:.2%} 速度: {speed:.2f} KB/s"
|
||
else:
|
||
msg = f"\r已下载: {downloaded_size} 字节 速度: {speed:.2f} KB/s"
|
||
|
||
print(msg, end="")
|
||
|
||
|
||
async def file_to_base64(file_path: str) -> str:
|
||
data_bytes = await asyncio.to_thread(Path(file_path).read_bytes)
|
||
base64_str = base64.b64encode(data_bytes).decode()
|
||
return "base64://" + base64_str
|
||
|
||
|
||
def get_local_ip_addresses():
|
||
net_interfaces = psutil.net_if_addrs()
|
||
network_ips = []
|
||
|
||
for interface, addrs in net_interfaces.items():
|
||
for addr in addrs:
|
||
if addr.family == socket.AF_INET: # 使用 socket.AF_INET 代替 psutil.AF_INET
|
||
network_ips.append(addr.address)
|
||
|
||
return network_ips
|
||
|
||
|
||
async def get_dashboard_version():
|
||
# First check user data directory (manually updated / downloaded dashboard).
|
||
dist_dir = os.path.join(get_astrbot_data_path(), "dist")
|
||
if not await asyncio.to_thread(os.path.exists, dist_dir):
|
||
# Fall back to the dist bundled inside the installed wheel.
|
||
_bundled = Path(get_astrbot_path()) / "astrbot" / "dashboard" / "dist"
|
||
if await asyncio.to_thread(_bundled.exists):
|
||
dist_dir = str(_bundled)
|
||
if await asyncio.to_thread(os.path.exists, dist_dir):
|
||
version_file = os.path.join(dist_dir, "assets", "version")
|
||
if await asyncio.to_thread(os.path.exists, version_file):
|
||
v = (
|
||
await asyncio.to_thread(Path(version_file).read_text, encoding="utf-8")
|
||
).strip()
|
||
return v
|
||
return None
|
||
|
||
|
||
async def download_dashboard(
|
||
path: str | None = None,
|
||
extract_path: str = "data",
|
||
latest: bool = True,
|
||
version: str | None = None,
|
||
proxy: str | None = None,
|
||
) -> None:
|
||
"""下载管理面板文件"""
|
||
if path is None:
|
||
zip_path = (
|
||
await asyncio.to_thread(Path(get_astrbot_data_path()).absolute)
|
||
/ "dashboard.zip"
|
||
)
|
||
else:
|
||
zip_path = await asyncio.to_thread(Path(path).absolute)
|
||
|
||
if latest or len(str(version)) != 40:
|
||
ver_name = "latest" if latest else version
|
||
dashboard_release_url = f"https://astrbot-registry.soulter.top/download/astrbot-dashboard/{ver_name}/dist.zip"
|
||
logger.info(
|
||
f"准备下载指定发行版本的 AstrBot WebUI 文件: {dashboard_release_url}",
|
||
)
|
||
try:
|
||
await download_file(
|
||
dashboard_release_url,
|
||
str(zip_path),
|
||
show_progress=True,
|
||
)
|
||
except BaseException as _:
|
||
if latest:
|
||
dashboard_release_url = "https://github.com/AstrBotDevs/AstrBot/releases/latest/download/dist.zip"
|
||
else:
|
||
dashboard_release_url = f"https://github.com/AstrBotDevs/AstrBot/releases/download/{version}/dist.zip"
|
||
if proxy:
|
||
dashboard_release_url = f"{proxy}/{dashboard_release_url}"
|
||
await download_file(
|
||
dashboard_release_url,
|
||
str(zip_path),
|
||
show_progress=True,
|
||
)
|
||
else:
|
||
url = f"https://github.com/AstrBotDevs/astrbot-release-harbour/releases/download/release-{version}/dist.zip"
|
||
logger.info(f"准备下载指定版本的 AstrBot WebUI: {url}")
|
||
if proxy:
|
||
url = f"{proxy}/{url}"
|
||
await download_file(url, str(zip_path), show_progress=True)
|
||
with zipfile.ZipFile(zip_path, "r") as z:
|
||
z.extractall(extract_path)
|