mirror of
https://github.com/xinnan-tech/xiaozhi-esp32-server.git
synced 2026-07-21 02:45:36 +00:00
refactor(get_news_from_newsnow): 改用流式处理解析网页内容
将原有的直接转换响应内容改为流式读取字节流并传入参数,适配MarkItDown的convert_stream接口,优化大内容处理时的内存占用
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import random
|
||||
import httpx
|
||||
from markitdown import MarkItDown
|
||||
from io import BytesIO
|
||||
from markitdown import MarkItDown, StreamInfo
|
||||
from config.logger import setup_logging
|
||||
from plugins_func.register import register_function, ToolType, ActionResponse, Action
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -144,7 +145,14 @@ async def fetch_news_detail(url):
|
||||
|
||||
# 使用MarkItDown清理HTML内容
|
||||
md = MarkItDown(enable_plugins=False)
|
||||
result = md.convert(response)
|
||||
result = md.convert_stream(
|
||||
BytesIO(response.content),
|
||||
stream_info=StreamInfo(
|
||||
mimetype="text/html",
|
||||
extension=".html",
|
||||
charset=response.encoding or "utf-8",
|
||||
),
|
||||
)
|
||||
|
||||
# 获取清理后的文本内容
|
||||
clean_text = result.text_content
|
||||
|
||||
Reference in New Issue
Block a user