diff --git a/util/agent/web_searcher.py b/util/agent/web_searcher.py index 7fa9e35c7..fdef04a09 100644 --- a/util/agent/web_searcher.py +++ b/util/agent/web_searcher.py @@ -53,10 +53,13 @@ async def search_from_bing(keyword: str) -> str: logger.info("web_searcher - search_from_bing: " + keyword) results = await google.search(keyword, 5) if len(results) == 0: + logger.debug("search google failed") results = await bing_search.search(keyword, 5) if len(results) == 0: + logger.debug("search bing failed") results = await sogo_search.search(keyword, 5) if len(results) == 0: + logger.debug("search sogo failed") return "没有搜索到结果" ret = "" idx = 1 diff --git a/util/search_engine_scraper/engine.py b/util/search_engine_scraper/engine.py index 9adbdea19..0146d8070 100644 --- a/util/search_engine_scraper/engine.py +++ b/util/search_engine_scraper/engine.py @@ -39,7 +39,6 @@ class SearchEngine(): headers = self.headers headers["Referer"] = url headers["User-Agent"] = random.choice(USER_AGENTS) - print(headers) if data: async with ClientSession() as session: async with session.post(url, headers=headers, data=data, timeout=self.TIMEOUT) as resp: diff --git a/util/search_engine_scraper/google.py b/util/search_engine_scraper/google.py index 03109883a..97aa6aa88 100644 --- a/util/search_engine_scraper/google.py +++ b/util/search_engine_scraper/google.py @@ -1,3 +1,4 @@ +import os from googlesearch import search try: @@ -10,12 +11,14 @@ except ImportError: from typing import List class Google(SearchEngine): + def __init__(self) -> None: + super().__init__() + self.proxy = os.environ.get("HTTPS_PROXY") async def search(self, query: str, num_results: int) -> List[SearchResult]: - index = 1 results = [] try: - ls = search(query, advanced=True, num_results=num_results, timeout=3) + ls = search(query, advanced=True, num_results=num_results, timeout=3, proxy=self.proxy) for i in ls: results.append(SearchResult(title=i.title, url=i.url, snippet=i.description)) except: