Compare commits
1 Commits
boyin_summ
...
frontier_3
| Author | SHA1 | Date | |
|---|---|---|---|
|
303d55c8a1 |
2
.github/workflows/build-with-latex-arm.yml
vendored
2
.github/workflows/build-with-latex-arm.yml
vendored
@@ -46,6 +46,6 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
push: true
|
push: true
|
||||||
platforms: linux/arm64
|
platforms: linux/arm64
|
||||||
file: docs/GithubAction+NoLocal+Latex
|
file: docs/GithubAction+NoLocal+Latex+Arm
|
||||||
tags: ${{ steps.meta.outputs.tags }}
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
labels: ${{ steps.meta.outputs.labels }}
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
@@ -1,36 +1,24 @@
|
|||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
def check_proxy(proxies, return_ip=False):
|
def check_proxy(proxies, return_ip=False):
|
||||||
"""
|
|
||||||
检查代理配置并返回结果。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
proxies (dict): 包含http和https代理配置的字典。
|
|
||||||
return_ip (bool, optional): 是否返回代理的IP地址。默认为False。
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str or None: 检查的结果信息或代理的IP地址（如果`return_ip`为True）。
|
|
||||||
"""
|
|
||||||
import requests
|
import requests
|
||||||
proxies_https = proxies['https'] if proxies is not None else '无'
|
proxies_https = proxies['https'] if proxies is not None else '无'
|
||||||
ip = None
|
ip = None
|
||||||
try:
|
try:
|
||||||
response = requests.get("https://ipapi.co/json/", proxies=proxies, timeout=4) # ⭐ 执行GET请求以获取代理信息
|
response = requests.get("https://ipapi.co/json/", proxies=proxies, timeout=4)
|
||||||
data = response.json()
|
data = response.json()
|
||||||
if 'country_name' in data:
|
if 'country_name' in data:
|
||||||
country = data['country_name']
|
country = data['country_name']
|
||||||
result = f"代理配置 {proxies_https}, 代理所在地：{country}"
|
result = f"代理配置 {proxies_https}, 代理所在地：{country}"
|
||||||
if 'ip' in data:
|
if 'ip' in data: ip = data['ip']
|
||||||
ip = data['ip']
|
|
||||||
elif 'error' in data:
|
elif 'error' in data:
|
||||||
alternative, ip = _check_with_backup_source(proxies) # ⭐ 调用备用方法检查代理配置
|
alternative, ip = _check_with_backup_source(proxies)
|
||||||
if alternative is None:
|
if alternative is None:
|
||||||
result = f"代理配置 {proxies_https}, 代理所在地：未知，IP查询频率受限"
|
result = f"代理配置 {proxies_https}, 代理所在地：未知，IP查询频率受限"
|
||||||
else:
|
else:
|
||||||
result = f"代理配置 {proxies_https}, 代理所在地：{alternative}"
|
result = f"代理配置 {proxies_https}, 代理所在地：{alternative}"
|
||||||
else:
|
else:
|
||||||
result = f"代理配置 {proxies_https}, 代理数据解析失败：{data}"
|
result = f"代理配置 {proxies_https}, 代理数据解析失败：{data}"
|
||||||
|
|
||||||
if not return_ip:
|
if not return_ip:
|
||||||
logger.warning(result)
|
logger.warning(result)
|
||||||
return result
|
return result
|
||||||
@@ -45,33 +33,17 @@ def check_proxy(proxies, return_ip=False):
|
|||||||
return ip
|
return ip
|
||||||
|
|
||||||
def _check_with_backup_source(proxies):
|
def _check_with_backup_source(proxies):
|
||||||
"""
|
|
||||||
通过备份源检查代理，并获取相应信息。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
proxies (dict): 包含代理信息的字典。
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
tuple: 代理信息(geo)和IP地址(ip)的元组。
|
|
||||||
"""
|
|
||||||
import random, string, requests
|
import random, string, requests
|
||||||
random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=32))
|
random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=32))
|
||||||
try:
|
try:
|
||||||
res_json = requests.get(f"http://{random_string}.edns.ip-api.com/json", proxies=proxies, timeout=4).json() # ⭐ 执行代理检查和备份源请求
|
res_json = requests.get(f"http://{random_string}.edns.ip-api.com/json", proxies=proxies, timeout=4).json()
|
||||||
return res_json['dns']['geo'], res_json['dns']['ip']
|
return res_json['dns']['geo'], res_json['dns']['ip']
|
||||||
except:
|
except:
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
def backup_and_download(current_version, remote_version):
|
def backup_and_download(current_version, remote_version):
|
||||||
"""
|
"""
|
||||||
一键更新协议：备份当前版本，下载远程版本并解压缩。
|
一键更新协议：备份和下载
|
||||||
|
|
||||||
Args:
|
|
||||||
current_version (str): 当前版本号。
|
|
||||||
remote_version (str): 远程版本号。
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: 新版本目录的路径。
|
|
||||||
"""
|
"""
|
||||||
from toolbox import get_conf
|
from toolbox import get_conf
|
||||||
import shutil
|
import shutil
|
||||||
@@ -88,7 +60,7 @@ def backup_and_download(current_version, remote_version):
|
|||||||
proxies = get_conf('proxies')
|
proxies = get_conf('proxies')
|
||||||
try: r = requests.get('https://github.com/binary-husky/chatgpt_academic/archive/refs/heads/master.zip', proxies=proxies, stream=True)
|
try: r = requests.get('https://github.com/binary-husky/chatgpt_academic/archive/refs/heads/master.zip', proxies=proxies, stream=True)
|
||||||
except: r = requests.get('https://public.agent-matrix.com/publish/master.zip', proxies=proxies, stream=True)
|
except: r = requests.get('https://public.agent-matrix.com/publish/master.zip', proxies=proxies, stream=True)
|
||||||
zip_file_path = backup_dir+'/master.zip' # ⭐ 保存备份文件的路径
|
zip_file_path = backup_dir+'/master.zip'
|
||||||
with open(zip_file_path, 'wb+') as f:
|
with open(zip_file_path, 'wb+') as f:
|
||||||
f.write(r.content)
|
f.write(r.content)
|
||||||
dst_path = new_version_dir
|
dst_path = new_version_dir
|
||||||
@@ -104,17 +76,6 @@ def backup_and_download(current_version, remote_version):
|
|||||||
def patch_and_restart(path):
|
def patch_and_restart(path):
|
||||||
"""
|
"""
|
||||||
一键更新协议：覆盖和重启
|
一键更新协议：覆盖和重启
|
||||||
|
|
||||||
Args:
|
|
||||||
path (str): 新版本代码所在的路径
|
|
||||||
|
|
||||||
注意事项:
|
|
||||||
如果您的程序没有使用config_private.py私密配置文件，则会将config.py重命名为config_private.py以避免配置丢失。
|
|
||||||
|
|
||||||
更新流程:
|
|
||||||
- 复制最新版本代码到当前目录
|
|
||||||
- 更新pip包依赖
|
|
||||||
- 如果更新失败，则提示手动安装依赖库并重启
|
|
||||||
"""
|
"""
|
||||||
from distutils import dir_util
|
from distutils import dir_util
|
||||||
import shutil
|
import shutil
|
||||||
@@ -123,43 +84,32 @@ def patch_and_restart(path):
|
|||||||
import time
|
import time
|
||||||
import glob
|
import glob
|
||||||
from shared_utils.colorful import log亮黄, log亮绿, log亮红
|
from shared_utils.colorful import log亮黄, log亮绿, log亮红
|
||||||
|
# if not using config_private, move origin config.py as config_private.py
|
||||||
if not os.path.exists('config_private.py'):
|
if not os.path.exists('config_private.py'):
|
||||||
log亮黄('由于您没有设置config_private.py私密配置，现将您的现有配置移动至config_private.py以防止配置丢失，',
|
log亮黄('由于您没有设置config_private.py私密配置，现将您的现有配置移动至config_private.py以防止配置丢失，',
|
||||||
'另外您可以随时在history子文件夹下找回旧版的程序。')
|
'另外您可以随时在history子文件夹下找回旧版的程序。')
|
||||||
shutil.copyfile('config.py', 'config_private.py')
|
shutil.copyfile('config.py', 'config_private.py')
|
||||||
|
|
||||||
path_new_version = glob.glob(path + '/*-master')[0]
|
path_new_version = glob.glob(path + '/*-master')[0]
|
||||||
dir_util.copy_tree(path_new_version, './') # ⭐ 将最新版本代码复制到当前目录
|
dir_util.copy_tree(path_new_version, './')
|
||||||
|
|
||||||
log亮绿('代码已经更新，即将更新pip包依赖……')
|
log亮绿('代码已经更新，即将更新pip包依赖……')
|
||||||
for i in reversed(range(5)): time.sleep(1); log亮绿(i)
|
for i in reversed(range(5)): time.sleep(1); log亮绿(i)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import subprocess
|
import subprocess
|
||||||
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-r', 'requirements.txt'])
|
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-r', 'requirements.txt'])
|
||||||
except:
|
except:
|
||||||
log亮红('pip包依赖安装出现问题，需要手动安装新增的依赖库 `python -m pip install -r requirements.txt`，然后在用常规的`python main.py`的方式启动。')
|
log亮红('pip包依赖安装出现问题，需要手动安装新增的依赖库 `python -m pip install -r requirements.txt`，然后在用常规的`python main.py`的方式启动。')
|
||||||
|
|
||||||
log亮绿('更新完成，您可以随时在history子文件夹下找回旧版的程序，5s之后重启')
|
log亮绿('更新完成，您可以随时在history子文件夹下找回旧版的程序，5s之后重启')
|
||||||
log亮红('假如重启失败，您可能需要手动安装新增的依赖库 `python -m pip install -r requirements.txt`，然后在用常规的`python main.py`的方式启动。')
|
log亮红('假如重启失败，您可能需要手动安装新增的依赖库 `python -m pip install -r requirements.txt`，然后在用常规的`python main.py`的方式启动。')
|
||||||
log亮绿(' ------------------------------ -----------------------------------')
|
log亮绿(' ------------------------------ -----------------------------------')
|
||||||
|
|
||||||
for i in reversed(range(8)): time.sleep(1); log亮绿(i)
|
for i in reversed(range(8)): time.sleep(1); log亮绿(i)
|
||||||
os.execl(sys.executable, sys.executable, *sys.argv) # 重启程序
|
os.execl(sys.executable, sys.executable, *sys.argv)
|
||||||
|
|
||||||
|
|
||||||
def get_current_version():
|
def get_current_version():
|
||||||
"""
|
|
||||||
获取当前的版本号。
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: 当前的版本号。如果无法获取版本号，则返回空字符串。
|
|
||||||
"""
|
|
||||||
import json
|
import json
|
||||||
try:
|
try:
|
||||||
with open('./version', 'r', encoding='utf8') as f:
|
with open('./version', 'r', encoding='utf8') as f:
|
||||||
current_version = json.loads(f.read())['version'] # ⭐ 从读取的json数据中提取版本号
|
current_version = json.loads(f.read())['version']
|
||||||
except:
|
except:
|
||||||
current_version = ""
|
current_version = ""
|
||||||
return current_version
|
return current_version
|
||||||
@@ -168,12 +118,6 @@ def get_current_version():
|
|||||||
def auto_update(raise_error=False):
|
def auto_update(raise_error=False):
|
||||||
"""
|
"""
|
||||||
一键更新协议：查询版本和用户意见
|
一键更新协议：查询版本和用户意见
|
||||||
|
|
||||||
Args:
|
|
||||||
raise_error (bool, optional): 是否在出错时抛出错误。默认为 False。
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
None
|
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
from toolbox import get_conf
|
from toolbox import get_conf
|
||||||
@@ -193,13 +137,13 @@ def auto_update(raise_error=False):
|
|||||||
current_version = json.loads(current_version)['version']
|
current_version = json.loads(current_version)['version']
|
||||||
if (remote_version - current_version) >= 0.01-1e-5:
|
if (remote_version - current_version) >= 0.01-1e-5:
|
||||||
from shared_utils.colorful import log亮黄
|
from shared_utils.colorful import log亮黄
|
||||||
log亮黄(f'\n新版本可用。新版本:{remote_version}，当前版本:{current_version}。{new_feature}') # ⭐ 在控制台打印新版本信息
|
log亮黄(f'\n新版本可用。新版本:{remote_version}，当前版本:{current_version}。{new_feature}')
|
||||||
logger.info('（1）Github更新地址:\nhttps://github.com/binary-husky/chatgpt_academic\n')
|
logger.info('（1）Github更新地址:\nhttps://github.com/binary-husky/chatgpt_academic\n')
|
||||||
user_instruction = input('（2）是否一键更新代码（Y+回车=确认，输入其他/无输入+回车=不更新）？')
|
user_instruction = input('（2）是否一键更新代码（Y+回车=确认，输入其他/无输入+回车=不更新）？')
|
||||||
if user_instruction in ['Y', 'y']:
|
if user_instruction in ['Y', 'y']:
|
||||||
path = backup_and_download(current_version, remote_version) # ⭐ 备份并下载文件
|
path = backup_and_download(current_version, remote_version)
|
||||||
try:
|
try:
|
||||||
patch_and_restart(path) # ⭐ 执行覆盖并重启操作
|
patch_and_restart(path)
|
||||||
except:
|
except:
|
||||||
msg = '更新失败。'
|
msg = '更新失败。'
|
||||||
if raise_error:
|
if raise_error:
|
||||||
@@ -219,9 +163,6 @@ def auto_update(raise_error=False):
|
|||||||
logger.info(msg)
|
logger.info(msg)
|
||||||
|
|
||||||
def warm_up_modules():
|
def warm_up_modules():
|
||||||
"""
|
|
||||||
预热模块，加载特定模块并执行预热操作。
|
|
||||||
"""
|
|
||||||
logger.info('正在执行一些模块的预热 ...')
|
logger.info('正在执行一些模块的预热 ...')
|
||||||
from toolbox import ProxyNetworkActivate
|
from toolbox import ProxyNetworkActivate
|
||||||
from request_llms.bridge_all import model_info
|
from request_llms.bridge_all import model_info
|
||||||
@@ -232,16 +173,6 @@ def warm_up_modules():
|
|||||||
enc.encode("模块预热", disallowed_special=())
|
enc.encode("模块预热", disallowed_special=())
|
||||||
|
|
||||||
def warm_up_vectordb():
|
def warm_up_vectordb():
|
||||||
"""
|
|
||||||
执行一些模块的预热操作。
|
|
||||||
|
|
||||||
本函数主要用于执行一些模块的预热操作，确保在后续的流程中能够顺利运行。
|
|
||||||
|
|
||||||
⭐ 关键作用：预热模块
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
None
|
|
||||||
"""
|
|
||||||
logger.info('正在执行一些模块的预热 ...')
|
logger.info('正在执行一些模块的预热 ...')
|
||||||
from toolbox import ProxyNetworkActivate
|
from toolbox import ProxyNetworkActivate
|
||||||
with ProxyNetworkActivate("Warmup_Modules"):
|
with ProxyNetworkActivate("Warmup_Modules"):
|
||||||
|
|||||||
@@ -21,13 +21,13 @@ def get_crazy_functions():
|
|||||||
from crazy_functions.询问多个大语言模型 import 同时问询
|
from crazy_functions.询问多个大语言模型 import 同时问询
|
||||||
from crazy_functions.SourceCode_Analyse import 解析一个Lua项目
|
from crazy_functions.SourceCode_Analyse import 解析一个Lua项目
|
||||||
from crazy_functions.SourceCode_Analyse import 解析一个CSharp项目
|
from crazy_functions.SourceCode_Analyse import 解析一个CSharp项目
|
||||||
|
from crazy_functions.总结word文档 import 总结word文档
|
||||||
from crazy_functions.解析JupyterNotebook import 解析ipynb文件
|
from crazy_functions.解析JupyterNotebook import 解析ipynb文件
|
||||||
from crazy_functions.Conversation_To_File import 载入对话历史存档
|
from crazy_functions.Conversation_To_File import 载入对话历史存档
|
||||||
from crazy_functions.Conversation_To_File import 对话历史存档
|
from crazy_functions.Conversation_To_File import 对话历史存档
|
||||||
from crazy_functions.Conversation_To_File import Conversation_To_File_Wrap
|
from crazy_functions.Conversation_To_File import Conversation_To_File_Wrap
|
||||||
from crazy_functions.Conversation_To_File import 删除所有本地对话历史记录
|
from crazy_functions.Conversation_To_File import 删除所有本地对话历史记录
|
||||||
from crazy_functions.辅助功能 import 清除缓存
|
from crazy_functions.辅助功能 import 清除缓存
|
||||||
from crazy_functions.批量文件询问 import 批量文件询问
|
|
||||||
from crazy_functions.Markdown_Translate import Markdown英译中
|
from crazy_functions.Markdown_Translate import Markdown英译中
|
||||||
from crazy_functions.批量总结PDF文档 import 批量总结PDF文档
|
from crazy_functions.批量总结PDF文档 import 批量总结PDF文档
|
||||||
from crazy_functions.PDF_Translate import 批量翻译PDF文档
|
from crazy_functions.PDF_Translate import 批量翻译PDF文档
|
||||||
@@ -49,7 +49,6 @@ def get_crazy_functions():
|
|||||||
from crazy_functions.Image_Generate import 图片生成_DALLE2, 图片生成_DALLE3, 图片修改_DALLE2
|
from crazy_functions.Image_Generate import 图片生成_DALLE2, 图片生成_DALLE3, 图片修改_DALLE2
|
||||||
from crazy_functions.Image_Generate_Wrap import ImageGen_Wrap
|
from crazy_functions.Image_Generate_Wrap import ImageGen_Wrap
|
||||||
from crazy_functions.SourceCode_Comment import 注释Python项目
|
from crazy_functions.SourceCode_Comment import 注释Python项目
|
||||||
from crazy_functions.SourceCode_Comment_Wrap import SourceCodeComment_Wrap
|
|
||||||
|
|
||||||
function_plugins = {
|
function_plugins = {
|
||||||
"虚空终端": {
|
"虚空终端": {
|
||||||
@@ -72,7 +71,6 @@ def get_crazy_functions():
|
|||||||
"AsButton": False,
|
"AsButton": False,
|
||||||
"Info": "上传一系列python源文件(或者压缩包), 为这些代码添加docstring | 输入参数为路径",
|
"Info": "上传一系列python源文件(或者压缩包), 为这些代码添加docstring | 输入参数为路径",
|
||||||
"Function": HotReload(注释Python项目),
|
"Function": HotReload(注释Python项目),
|
||||||
"Class": SourceCodeComment_Wrap,
|
|
||||||
},
|
},
|
||||||
"载入对话历史存档（先上传存档或输入路径）": {
|
"载入对话历史存档（先上传存档或输入路径）": {
|
||||||
"Group": "对话",
|
"Group": "对话",
|
||||||
@@ -110,13 +108,12 @@ def get_crazy_functions():
|
|||||||
"Function": HotReload(Latex翻译中文并重新编译PDF), # 当注册Class后，Function旧接口仅会在“虚空终端”中起作用
|
"Function": HotReload(Latex翻译中文并重新编译PDF), # 当注册Class后，Function旧接口仅会在“虚空终端”中起作用
|
||||||
"Class": Arxiv_Localize, # 新一代插件需要注册Class
|
"Class": Arxiv_Localize, # 新一代插件需要注册Class
|
||||||
},
|
},
|
||||||
"批量文件询问": {
|
"批量总结Word文档": {
|
||||||
"Group": "学术",
|
"Group": "学术",
|
||||||
"Color": "stop",
|
"Color": "stop",
|
||||||
"AsButton": False,
|
"AsButton": False,
|
||||||
"AdvancedArgs": True,
|
"Info": "批量总结word文档 | 输入参数为路径",
|
||||||
"Info": "通过在高级参数区写入prompt，可自定义询问逻辑，默认情况下为总结逻辑 | 输入参数为路径",
|
"Function": HotReload(总结word文档),
|
||||||
"Function": HotReload(批量文件询问),
|
|
||||||
},
|
},
|
||||||
"解析整个Matlab项目": {
|
"解析整个Matlab项目": {
|
||||||
"Group": "编程",
|
"Group": "编程",
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from toolbox import CatchException, report_exception, update_ui_lastest_msg, zip
|
|||||||
from functools import partial
|
from functools import partial
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
import glob, os, requests, time, json, tarfile, threading
|
import glob, os, requests, time, json, tarfile
|
||||||
|
|
||||||
pj = os.path.join
|
pj = os.path.join
|
||||||
ARXIV_CACHE_DIR = get_conf("ARXIV_CACHE_DIR")
|
ARXIV_CACHE_DIR = get_conf("ARXIV_CACHE_DIR")
|
||||||
@@ -338,17 +338,11 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
|
|||||||
# <-------------- more requirements ------------->
|
# <-------------- more requirements ------------->
|
||||||
if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg")
|
if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg")
|
||||||
more_req = plugin_kwargs.get("advanced_arg", "")
|
more_req = plugin_kwargs.get("advanced_arg", "")
|
||||||
|
no_cache = more_req.startswith("--no-cache")
|
||||||
no_cache = ("--no-cache" in more_req)
|
if no_cache: more_req.lstrip("--no-cache")
|
||||||
if no_cache: more_req = more_req.replace("--no-cache", "").strip()
|
|
||||||
|
|
||||||
allow_gptac_cloud_io = ("--allow-cloudio" in more_req) # 从云端下载翻译结果，以及上传翻译结果到云端
|
|
||||||
if allow_gptac_cloud_io: more_req = more_req.replace("--allow-cloudio", "").strip()
|
|
||||||
|
|
||||||
allow_cache = not no_cache
|
allow_cache = not no_cache
|
||||||
_switch_prompt_ = partial(switch_prompt, more_requirement=more_req)
|
_switch_prompt_ = partial(switch_prompt, more_requirement=more_req)
|
||||||
|
|
||||||
|
|
||||||
# <-------------- check deps ------------->
|
# <-------------- check deps ------------->
|
||||||
try:
|
try:
|
||||||
import glob, os, time, subprocess
|
import glob, os, time, subprocess
|
||||||
@@ -375,20 +369,6 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
|
|||||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
return
|
return
|
||||||
|
|
||||||
# #################################################################
|
|
||||||
if allow_gptac_cloud_io and arxiv_id:
|
|
||||||
# 访问 GPTAC学术云，查询云端是否存在该论文的翻译版本
|
|
||||||
from crazy_functions.latex_fns.latex_actions import check_gptac_cloud
|
|
||||||
success, downloaded = check_gptac_cloud(arxiv_id, chatbot)
|
|
||||||
if success:
|
|
||||||
chatbot.append([
|
|
||||||
f"检测到GPTAC云端存在翻译版本, 如果不满意翻译结果, 请禁用云端分享, 然后重新执行。",
|
|
||||||
None
|
|
||||||
])
|
|
||||||
yield from update_ui(chatbot=chatbot, history=history)
|
|
||||||
return
|
|
||||||
#################################################################
|
|
||||||
|
|
||||||
if os.path.exists(txt):
|
if os.path.exists(txt):
|
||||||
project_folder = txt
|
project_folder = txt
|
||||||
else:
|
else:
|
||||||
@@ -426,21 +406,14 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
|
|||||||
# <-------------- zip PDF ------------->
|
# <-------------- zip PDF ------------->
|
||||||
zip_res = zip_result(project_folder)
|
zip_res = zip_result(project_folder)
|
||||||
if success:
|
if success:
|
||||||
if allow_gptac_cloud_io and arxiv_id:
|
|
||||||
# 如果用户允许，我们将翻译好的arxiv论文PDF上传到GPTAC学术云
|
|
||||||
from crazy_functions.latex_fns.latex_actions import upload_to_gptac_cloud_if_user_allow
|
|
||||||
threading.Thread(target=upload_to_gptac_cloud_if_user_allow,
|
|
||||||
args=(chatbot, arxiv_id), daemon=True).start()
|
|
||||||
|
|
||||||
chatbot.append((f"成功啦", '请查收结果（压缩包）...'))
|
chatbot.append((f"成功啦", '请查收结果（压缩包）...'))
|
||||||
yield from update_ui(chatbot=chatbot, history=history)
|
yield from update_ui(chatbot=chatbot, history=history);
|
||||||
time.sleep(1) # 刷新界面
|
time.sleep(1) # 刷新界面
|
||||||
promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
|
promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
chatbot.append((f"失败了",
|
chatbot.append((f"失败了",
|
||||||
'虽然PDF生成失败了, 但请查收结果（压缩包）, 内含已经翻译的Tex文档, 您可以到Github Issue区, 用该压缩包进行反馈。如系统是Linux，请检查系统字体（见Github wiki） ...'))
|
'虽然PDF生成失败了, 但请查收结果（压缩包）, 内含已经翻译的Tex文档, 您可以到Github Issue区, 用该压缩包进行反馈。如系统是Linux，请检查系统字体（见Github wiki） ...'))
|
||||||
yield from update_ui(chatbot=chatbot, history=history)
|
yield from update_ui(chatbot=chatbot, history=history);
|
||||||
time.sleep(1) # 刷新界面
|
time.sleep(1) # 刷新界面
|
||||||
promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
|
promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
|
||||||
|
|
||||||
|
|||||||
@@ -30,8 +30,6 @@ class Arxiv_Localize(GptAcademicPluginTemplate):
|
|||||||
default_value="", type="string").model_dump_json(), # 高级参数输入区，自动同步
|
default_value="", type="string").model_dump_json(), # 高级参数输入区，自动同步
|
||||||
"allow_cache":
|
"allow_cache":
|
||||||
ArgProperty(title="是否允许从缓存中调取结果", options=["允许缓存", "从头执行"], default_value="允许缓存", description="无", type="dropdown").model_dump_json(),
|
ArgProperty(title="是否允许从缓存中调取结果", options=["允许缓存", "从头执行"], default_value="允许缓存", description="无", type="dropdown").model_dump_json(),
|
||||||
"allow_cloudio":
|
|
||||||
ArgProperty(title="是否允许从GPTAC学术云下载(或者上传)翻译结果(仅针对Arxiv论文)", options=["允许", "禁止"], default_value="禁止", description="共享文献，互助互利", type="dropdown").model_dump_json(),
|
|
||||||
}
|
}
|
||||||
return gui_definition
|
return gui_definition
|
||||||
|
|
||||||
@@ -40,14 +38,9 @@ class Arxiv_Localize(GptAcademicPluginTemplate):
|
|||||||
执行插件
|
执行插件
|
||||||
"""
|
"""
|
||||||
allow_cache = plugin_kwargs["allow_cache"]
|
allow_cache = plugin_kwargs["allow_cache"]
|
||||||
allow_cloudio = plugin_kwargs["allow_cloudio"]
|
|
||||||
advanced_arg = plugin_kwargs["advanced_arg"]
|
advanced_arg = plugin_kwargs["advanced_arg"]
|
||||||
|
|
||||||
if allow_cache == "从头执行": plugin_kwargs["advanced_arg"] = "--no-cache " + plugin_kwargs["advanced_arg"]
|
if allow_cache == "从头执行": plugin_kwargs["advanced_arg"] = "--no-cache " + plugin_kwargs["advanced_arg"]
|
||||||
|
|
||||||
# 从云端下载翻译结果，以及上传翻译结果到云端；人人为我，我为人人。
|
|
||||||
if allow_cloudio == "允许": plugin_kwargs["advanced_arg"] = "--allow-cloudio " + plugin_kwargs["advanced_arg"]
|
|
||||||
|
|
||||||
yield from Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request)
|
yield from Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
|||||||
pfg.file_contents.append(file_content)
|
pfg.file_contents.append(file_content)
|
||||||
|
|
||||||
# <-------- 拆分过长的Markdown文件 ---------->
|
# <-------- 拆分过长的Markdown文件 ---------->
|
||||||
pfg.run_file_split(max_token_limit=1024)
|
pfg.run_file_split(max_token_limit=2048)
|
||||||
n_split = len(pfg.sp_file_contents)
|
n_split = len(pfg.sp_file_contents)
|
||||||
|
|
||||||
# <-------- 多线程翻译开始 ---------->
|
# <-------- 多线程翻译开始 ---------->
|
||||||
|
|||||||
@@ -6,10 +6,7 @@ from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_ver
|
|||||||
from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||||
from crazy_functions.agent_fns.python_comment_agent import PythonCodeComment
|
from crazy_functions.agent_fns.python_comment_agent import PythonCodeComment
|
||||||
from crazy_functions.diagram_fns.file_tree import FileNode
|
from crazy_functions.diagram_fns.file_tree import FileNode
|
||||||
from crazy_functions.agent_fns.watchdog import WatchDog
|
|
||||||
from shared_utils.advanced_markdown_format import markdown_convertion_for_file
|
from shared_utils.advanced_markdown_format import markdown_convertion_for_file
|
||||||
from loguru import logger
|
|
||||||
|
|
||||||
|
|
||||||
def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
|
def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
|
||||||
|
|
||||||
@@ -27,13 +24,12 @@ def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
|||||||
file_tree_struct.add_file(file_path, file_path)
|
file_tree_struct.add_file(file_path, file_path)
|
||||||
|
|
||||||
# <第一步，逐个文件分析，多线程>
|
# <第一步，逐个文件分析，多线程>
|
||||||
lang = "" if not plugin_kwargs["use_chinese"] else " (you must use Chinese)"
|
|
||||||
for index, fp in enumerate(file_manifest):
|
for index, fp in enumerate(file_manifest):
|
||||||
# 读取文件
|
# 读取文件
|
||||||
with open(fp, 'r', encoding='utf-8', errors='replace') as f:
|
with open(fp, 'r', encoding='utf-8', errors='replace') as f:
|
||||||
file_content = f.read()
|
file_content = f.read()
|
||||||
prefix = ""
|
prefix = ""
|
||||||
i_say = prefix + f'Please conclude the following source code at {os.path.relpath(fp, project_folder)} with only one sentence{lang}, the code is:\n```{file_content}```'
|
i_say = prefix + f'Please conclude the following source code at {os.path.relpath(fp, project_folder)} with only one sentence, the code is:\n```{file_content}```'
|
||||||
i_say_show_user = prefix + f'[{index+1}/{len(file_manifest)}] 请用一句话对下面的程序文件做一个整体概述: {fp}'
|
i_say_show_user = prefix + f'[{index+1}/{len(file_manifest)}] 请用一句话对下面的程序文件做一个整体概述: {fp}'
|
||||||
# 装载请求内容
|
# 装载请求内容
|
||||||
MAX_TOKEN_SINGLE_FILE = 2560
|
MAX_TOKEN_SINGLE_FILE = 2560
|
||||||
@@ -41,7 +37,7 @@ def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
|||||||
inputs_array.append(i_say)
|
inputs_array.append(i_say)
|
||||||
inputs_show_user_array.append(i_say_show_user)
|
inputs_show_user_array.append(i_say_show_user)
|
||||||
history_array.append([])
|
history_array.append([])
|
||||||
sys_prompt_array.append(f"You are a software architecture analyst analyzing a source code project. Do not dig into details, tell me what the code is doing in general. Your answer must be short, simple and clear{lang}.")
|
sys_prompt_array.append("You are a software architecture analyst analyzing a source code project. Do not dig into details, tell me what the code is doing in general. Your answer must be short, simple and clear.")
|
||||||
# 文件读取完成，对每一个源代码文件，生成一个请求线程，发送到大模型进行分析
|
# 文件读取完成，对每一个源代码文件，生成一个请求线程，发送到大模型进行分析
|
||||||
gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
||||||
inputs_array = inputs_array,
|
inputs_array = inputs_array,
|
||||||
@@ -54,20 +50,10 @@ def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
|||||||
)
|
)
|
||||||
|
|
||||||
# <第二步，逐个文件分析，生成带注释文件>
|
# <第二步，逐个文件分析，生成带注释文件>
|
||||||
tasks = ["" for _ in range(len(file_manifest))]
|
|
||||||
def bark_fn(tasks):
|
|
||||||
for i in range(len(tasks)): tasks[i] = "watchdog is dead"
|
|
||||||
wd = WatchDog(timeout=10, bark_fn=lambda: bark_fn(tasks), interval=3, msg="ThreadWatcher timeout")
|
|
||||||
wd.begin_watch()
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
executor = ThreadPoolExecutor(max_workers=get_conf('DEFAULT_WORKER_NUM'))
|
executor = ThreadPoolExecutor(max_workers=get_conf('DEFAULT_WORKER_NUM'))
|
||||||
def _task_multi_threading(i_say, gpt_say, fp, file_tree_struct, index):
|
def _task_multi_threading(i_say, gpt_say, fp, file_tree_struct):
|
||||||
language = 'Chinese' if plugin_kwargs["use_chinese"] else 'English'
|
pcc = PythonCodeComment(llm_kwargs, language='English')
|
||||||
def observe_window_update(x):
|
|
||||||
if tasks[index] == "watchdog is dead":
|
|
||||||
raise TimeoutError("ThreadWatcher: watchdog is dead")
|
|
||||||
tasks[index] = x
|
|
||||||
pcc = PythonCodeComment(llm_kwargs, plugin_kwargs, language=language, observe_window_update=observe_window_update)
|
|
||||||
pcc.read_file(path=fp, brief=gpt_say)
|
pcc.read_file(path=fp, brief=gpt_say)
|
||||||
revised_path, revised_content = pcc.begin_comment_source_code(None, None)
|
revised_path, revised_content = pcc.begin_comment_source_code(None, None)
|
||||||
file_tree_struct.manifest[fp].revised_path = revised_path
|
file_tree_struct.manifest[fp].revised_path = revised_path
|
||||||
@@ -79,8 +65,7 @@ def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
|||||||
with open("crazy_functions/agent_fns/python_comment_compare.html", 'r', encoding='utf-8') as f:
|
with open("crazy_functions/agent_fns/python_comment_compare.html", 'r', encoding='utf-8') as f:
|
||||||
html_template = f.read()
|
html_template = f.read()
|
||||||
warp = lambda x: "```python\n\n" + x + "\n\n```"
|
warp = lambda x: "```python\n\n" + x + "\n\n```"
|
||||||
from themes.theme import load_dynamic_theme
|
from themes.theme import advanced_css
|
||||||
_, advanced_css, _, _ = load_dynamic_theme("Default")
|
|
||||||
html_template = html_template.replace("ADVANCED_CSS", advanced_css)
|
html_template = html_template.replace("ADVANCED_CSS", advanced_css)
|
||||||
html_template = html_template.replace("REPLACE_CODE_FILE_LEFT", pcc.get_markdown_block_in_html(markdown_convertion_for_file(warp(pcc.original_content))))
|
html_template = html_template.replace("REPLACE_CODE_FILE_LEFT", pcc.get_markdown_block_in_html(markdown_convertion_for_file(warp(pcc.original_content))))
|
||||||
html_template = html_template.replace("REPLACE_CODE_FILE_RIGHT", pcc.get_markdown_block_in_html(markdown_convertion_for_file(warp(revised_content))))
|
html_template = html_template.replace("REPLACE_CODE_FILE_RIGHT", pcc.get_markdown_block_in_html(markdown_convertion_for_file(warp(revised_content))))
|
||||||
@@ -88,21 +73,17 @@ def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
|||||||
file_tree_struct.manifest[fp].compare_html = compare_html_path
|
file_tree_struct.manifest[fp].compare_html = compare_html_path
|
||||||
with open(compare_html_path, 'w', encoding='utf-8') as f:
|
with open(compare_html_path, 'w', encoding='utf-8') as f:
|
||||||
f.write(html_template)
|
f.write(html_template)
|
||||||
tasks[index] = ""
|
# print('done 1')
|
||||||
|
|
||||||
chatbot.append([None, f"正在处理:"])
|
chatbot.append([None, f"正在处理:"])
|
||||||
futures = []
|
futures = []
|
||||||
index = 0
|
|
||||||
for i_say, gpt_say, fp in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], file_manifest):
|
for i_say, gpt_say, fp in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], file_manifest):
|
||||||
future = executor.submit(_task_multi_threading, i_say, gpt_say, fp, file_tree_struct, index)
|
future = executor.submit(_task_multi_threading, i_say, gpt_say, fp, file_tree_struct)
|
||||||
index += 1
|
|
||||||
futures.append(future)
|
futures.append(future)
|
||||||
|
|
||||||
# <第三步，等待任务完成>
|
|
||||||
cnt = 0
|
cnt = 0
|
||||||
while True:
|
while True:
|
||||||
cnt += 1
|
cnt += 1
|
||||||
wd.feed()
|
|
||||||
time.sleep(3)
|
time.sleep(3)
|
||||||
worker_done = [h.done() for h in futures]
|
worker_done = [h.done() for h in futures]
|
||||||
remain = len(worker_done) - sum(worker_done)
|
remain = len(worker_done) - sum(worker_done)
|
||||||
@@ -111,18 +92,14 @@ def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
|||||||
preview_html_list = []
|
preview_html_list = []
|
||||||
for done, fp in zip(worker_done, file_manifest):
|
for done, fp in zip(worker_done, file_manifest):
|
||||||
if not done: continue
|
if not done: continue
|
||||||
if hasattr(file_tree_struct.manifest[fp], 'compare_html'):
|
preview_html_list.append(file_tree_struct.manifest[fp].compare_html)
|
||||||
preview_html_list.append(file_tree_struct.manifest[fp].compare_html)
|
|
||||||
else:
|
|
||||||
logger.error(f"文件: {fp} 的注释结果未能成功")
|
|
||||||
file_links = generate_file_link(preview_html_list)
|
file_links = generate_file_link(preview_html_list)
|
||||||
|
|
||||||
yield from update_ui_lastest_msg(
|
yield from update_ui_lastest_msg(
|
||||||
f"当前任务: <br/>{'<br/>'.join(tasks)}.<br/>" +
|
f"剩余源文件数量: {remain}.\n\n" +
|
||||||
f"剩余源文件数量: {remain}.<br/>" +
|
f"已完成的文件: {sum(worker_done)}.\n\n" +
|
||||||
f"已完成的文件: {sum(worker_done)}.<br/>" +
|
|
||||||
file_links +
|
file_links +
|
||||||
"<br/>" +
|
"\n\n" +
|
||||||
''.join(['.']*(cnt % 10 + 1)
|
''.join(['.']*(cnt % 10 + 1)
|
||||||
), chatbot=chatbot, history=history, delay=0)
|
), chatbot=chatbot, history=history, delay=0)
|
||||||
yield from update_ui(chatbot=chatbot, history=[]) # 刷新界面
|
yield from update_ui(chatbot=chatbot, history=[]) # 刷新界面
|
||||||
@@ -143,7 +120,6 @@ def 注释源代码(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
|||||||
@CatchException
|
@CatchException
|
||||||
def 注释Python项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request):
|
def 注释Python项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request):
|
||||||
history = [] # 清空历史，以免输入溢出
|
history = [] # 清空历史，以免输入溢出
|
||||||
plugin_kwargs["use_chinese"] = plugin_kwargs.get("use_chinese", False)
|
|
||||||
import glob, os
|
import glob, os
|
||||||
if os.path.exists(txt):
|
if os.path.exists(txt):
|
||||||
project_folder = txt
|
project_folder = txt
|
||||||
|
|||||||
@@ -1,36 +0,0 @@
|
|||||||
|
|
||||||
from toolbox import get_conf, update_ui
|
|
||||||
from crazy_functions.plugin_template.plugin_class_template import GptAcademicPluginTemplate, ArgProperty
|
|
||||||
from crazy_functions.SourceCode_Comment import 注释Python项目
|
|
||||||
|
|
||||||
class SourceCodeComment_Wrap(GptAcademicPluginTemplate):
|
|
||||||
def __init__(self):
|
|
||||||
"""
|
|
||||||
请注意`execute`会执行在不同的线程中，因此您在定义和使用类变量时，应当慎之又慎！
|
|
||||||
"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
def define_arg_selection_menu(self):
|
|
||||||
"""
|
|
||||||
定义插件的二级选项菜单
|
|
||||||
"""
|
|
||||||
gui_definition = {
|
|
||||||
"main_input":
|
|
||||||
ArgProperty(title="路径", description="程序路径（上传文件后自动填写）", default_value="", type="string").model_dump_json(), # 主输入，自动从输入框同步
|
|
||||||
"use_chinese":
|
|
||||||
ArgProperty(title="注释语言", options=["英文", "中文"], default_value="英文", description="无", type="dropdown").model_dump_json(),
|
|
||||||
# "use_emoji":
|
|
||||||
# ArgProperty(title="在注释中使用emoji", options=["禁止", "允许"], default_value="禁止", description="无", type="dropdown").model_dump_json(),
|
|
||||||
}
|
|
||||||
return gui_definition
|
|
||||||
|
|
||||||
def execute(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request):
|
|
||||||
"""
|
|
||||||
执行插件
|
|
||||||
"""
|
|
||||||
if plugin_kwargs["use_chinese"] == "中文":
|
|
||||||
plugin_kwargs["use_chinese"] = True
|
|
||||||
else:
|
|
||||||
plugin_kwargs["use_chinese"] = False
|
|
||||||
|
|
||||||
yield from 注释Python项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request)
|
|
||||||
@@ -68,7 +68,6 @@ Be aware:
|
|||||||
1. You must NOT modify the indent of code.
|
1. You must NOT modify the indent of code.
|
||||||
2. You are NOT authorized to change or translate non-comment code, and you are NOT authorized to add empty lines either, toggle qu.
|
2. You are NOT authorized to change or translate non-comment code, and you are NOT authorized to add empty lines either, toggle qu.
|
||||||
3. Use {LANG} to add comments and docstrings. Do NOT translate Chinese that is already in the code.
|
3. Use {LANG} to add comments and docstrings. Do NOT translate Chinese that is already in the code.
|
||||||
4. Besides adding a docstring, use the ⭐ symbol to annotate the most core and important line of code within the function, explaining its role.
|
|
||||||
|
|
||||||
------------------ Example ------------------
|
------------------ Example ------------------
|
||||||
INPUT:
|
INPUT:
|
||||||
@@ -117,66 +116,10 @@ def zip_result(folder):
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
|
|
||||||
revise_funtion_prompt_chinese = '''
|
|
||||||
您需要阅读以下代码，并根据以下说明修订源代码({FILE_BASENAME}):
|
|
||||||
1. 如果源代码中包含函数的话, 你应该分析给定函数实现了什么功能
|
|
||||||
2. 如果源代码中包含函数的话, 你需要为函数添加docstring, docstring必须使用中文
|
|
||||||
|
|
||||||
请注意：
|
|
||||||
1. 你不得修改代码的缩进
|
|
||||||
2. 你无权更改或翻译代码中的非注释部分，也不允许添加空行
|
|
||||||
3. 使用 {LANG} 添加注释和文档字符串。不要翻译代码中已有的中文
|
|
||||||
4. 除了添加docstring之外, 使用⭐符号给该函数中最核心、最重要的一行代码添加注释，并说明其作用
|
|
||||||
|
|
||||||
------------------ 示例 ------------------
|
|
||||||
INPUT:
|
|
||||||
```
|
|
||||||
L0000 |
|
|
||||||
L0001 |def zip_result(folder):
|
|
||||||
L0002 | t = gen_time_str()
|
|
||||||
L0003 | zip_folder(folder, get_log_folder(), f"result.zip")
|
|
||||||
L0004 | return os.path.join(get_log_folder(), f"result.zip")
|
|
||||||
L0005 |
|
|
||||||
L0006 |
|
|
||||||
```
|
|
||||||
|
|
||||||
OUTPUT:
|
|
||||||
|
|
||||||
<instruction_1_purpose>
|
|
||||||
该函数用于压缩指定文件夹，并返回生成的`zip`文件的路径。
|
|
||||||
</instruction_1_purpose>
|
|
||||||
<instruction_2_revised_code>
|
|
||||||
```
|
|
||||||
def zip_result(folder):
|
|
||||||
"""
|
|
||||||
该函数将指定的文件夹压缩成ZIP文件, 并将其存储在日志文件夹中。
|
|
||||||
|
|
||||||
输入参数:
|
|
||||||
folder (str): 需要压缩的文件夹的路径。
|
|
||||||
返回值:
|
|
||||||
str: 日志文件夹中创建的ZIP文件的路径。
|
|
||||||
"""
|
|
||||||
t = gen_time_str()
|
|
||||||
zip_folder(folder, get_log_folder(), f"result.zip") # ⭐ 执行文件夹的压缩
|
|
||||||
return os.path.join(get_log_folder(), f"result.zip")
|
|
||||||
```
|
|
||||||
</instruction_2_revised_code>
|
|
||||||
------------------ End of Example ------------------
|
|
||||||
|
|
||||||
|
|
||||||
------------------ the real INPUT you need to process NOW ({FILE_BASENAME}) ------------------
|
|
||||||
```
|
|
||||||
{THE_CODE}
|
|
||||||
```
|
|
||||||
{INDENT_REMINDER}
|
|
||||||
{BRIEF_REMINDER}
|
|
||||||
{HINT_REMINDER}
|
|
||||||
'''
|
|
||||||
|
|
||||||
|
|
||||||
class PythonCodeComment():
|
class PythonCodeComment():
|
||||||
|
|
||||||
def __init__(self, llm_kwargs, plugin_kwargs, language, observe_window_update) -> None:
|
def __init__(self, llm_kwargs, language) -> None:
|
||||||
self.original_content = ""
|
self.original_content = ""
|
||||||
self.full_context = []
|
self.full_context = []
|
||||||
self.full_context_with_line_no = []
|
self.full_context_with_line_no = []
|
||||||
@@ -184,13 +127,7 @@ class PythonCodeComment():
|
|||||||
self.page_limit = 100 # 100 lines of code each page
|
self.page_limit = 100 # 100 lines of code each page
|
||||||
self.ignore_limit = 20
|
self.ignore_limit = 20
|
||||||
self.llm_kwargs = llm_kwargs
|
self.llm_kwargs = llm_kwargs
|
||||||
self.plugin_kwargs = plugin_kwargs
|
|
||||||
self.language = language
|
self.language = language
|
||||||
self.observe_window_update = observe_window_update
|
|
||||||
if self.language == "chinese":
|
|
||||||
self.core_prompt = revise_funtion_prompt_chinese
|
|
||||||
else:
|
|
||||||
self.core_prompt = revise_funtion_prompt
|
|
||||||
self.path = None
|
self.path = None
|
||||||
self.file_basename = None
|
self.file_basename = None
|
||||||
self.file_brief = ""
|
self.file_brief = ""
|
||||||
@@ -321,7 +258,7 @@ class PythonCodeComment():
|
|||||||
hint_reminder = "" if hint is None else f"(Reminder: do not ignore or modify code such as `{hint}`, provide complete code in the OUTPUT.)"
|
hint_reminder = "" if hint is None else f"(Reminder: do not ignore or modify code such as `{hint}`, provide complete code in the OUTPUT.)"
|
||||||
self.llm_kwargs['temperature'] = 0
|
self.llm_kwargs['temperature'] = 0
|
||||||
result = predict_no_ui_long_connection(
|
result = predict_no_ui_long_connection(
|
||||||
inputs=self.core_prompt.format(
|
inputs=revise_funtion_prompt.format(
|
||||||
LANG=self.language,
|
LANG=self.language,
|
||||||
FILE_BASENAME=self.file_basename,
|
FILE_BASENAME=self.file_basename,
|
||||||
THE_CODE=code,
|
THE_CODE=code,
|
||||||
@@ -411,7 +348,6 @@ class PythonCodeComment():
|
|||||||
try:
|
try:
|
||||||
# yield from update_ui_lastest_msg(f"({self.file_basename}) 正在读取下一段代码片段:\n", chatbot=chatbot, history=history, delay=0)
|
# yield from update_ui_lastest_msg(f"({self.file_basename}) 正在读取下一段代码片段:\n", chatbot=chatbot, history=history, delay=0)
|
||||||
next_batch, line_no_start, line_no_end = self.get_next_batch()
|
next_batch, line_no_start, line_no_end = self.get_next_batch()
|
||||||
self.observe_window_update(f"正在处理{self.file_basename} - {line_no_start}/{len(self.full_context)}\n")
|
|
||||||
# yield from update_ui_lastest_msg(f"({self.file_basename}) 处理代码片段:\n\n{next_batch}", chatbot=chatbot, history=history, delay=0)
|
# yield from update_ui_lastest_msg(f"({self.file_basename}) 处理代码片段:\n\n{next_batch}", chatbot=chatbot, history=history, delay=0)
|
||||||
|
|
||||||
hint = None
|
hint = None
|
||||||
|
|||||||
@@ -1,47 +1,39 @@
|
|||||||
import token
|
import ast
|
||||||
import tokenize
|
|
||||||
import copy
|
class CommentRemover(ast.NodeTransformer):
|
||||||
import io
|
def visit_FunctionDef(self, node):
|
||||||
|
# 移除函数的文档字符串
|
||||||
|
if (node.body and isinstance(node.body[0], ast.Expr) and
|
||||||
|
isinstance(node.body[0].value, ast.Str)):
|
||||||
|
node.body = node.body[1:]
|
||||||
|
self.generic_visit(node)
|
||||||
|
return node
|
||||||
|
|
||||||
|
def visit_ClassDef(self, node):
|
||||||
|
# 移除类的文档字符串
|
||||||
|
if (node.body and isinstance(node.body[0], ast.Expr) and
|
||||||
|
isinstance(node.body[0].value, ast.Str)):
|
||||||
|
node.body = node.body[1:]
|
||||||
|
self.generic_visit(node)
|
||||||
|
return node
|
||||||
|
|
||||||
|
def visit_Module(self, node):
|
||||||
|
# 移除模块的文档字符串
|
||||||
|
if (node.body and isinstance(node.body[0], ast.Expr) and
|
||||||
|
isinstance(node.body[0].value, ast.Str)):
|
||||||
|
node.body = node.body[1:]
|
||||||
|
self.generic_visit(node)
|
||||||
|
return node
|
||||||
|
|
||||||
|
|
||||||
def remove_python_comments(input_source: str) -> str:
|
def remove_python_comments(source_code):
|
||||||
source_flag = copy.copy(input_source)
|
# 解析源代码为 AST
|
||||||
source = io.StringIO(input_source)
|
tree = ast.parse(source_code)
|
||||||
ls = input_source.split('\n')
|
# 移除注释
|
||||||
prev_toktype = token.INDENT
|
transformer = CommentRemover()
|
||||||
readline = source.readline
|
tree = transformer.visit(tree)
|
||||||
|
# 将处理后的 AST 转换回源代码
|
||||||
def get_char_index(lineno, col):
|
return ast.unparse(tree)
|
||||||
# find the index of the char in the source code
|
|
||||||
if lineno == 1:
|
|
||||||
return len('\n'.join(ls[:(lineno-1)])) + col
|
|
||||||
else:
|
|
||||||
return len('\n'.join(ls[:(lineno-1)])) + col + 1
|
|
||||||
|
|
||||||
def replace_char_between(start_lineno, start_col, end_lineno, end_col, source, replace_char, ls):
|
|
||||||
# replace char between start_lineno, start_col and end_lineno, end_col with replace_char, but keep '\n' and ' '
|
|
||||||
b = get_char_index(start_lineno, start_col)
|
|
||||||
e = get_char_index(end_lineno, end_col)
|
|
||||||
for i in range(b, e):
|
|
||||||
if source[i] == '\n':
|
|
||||||
source = source[:i] + '\n' + source[i+1:]
|
|
||||||
elif source[i] == ' ':
|
|
||||||
source = source[:i] + ' ' + source[i+1:]
|
|
||||||
else:
|
|
||||||
source = source[:i] + replace_char + source[i+1:]
|
|
||||||
return source
|
|
||||||
|
|
||||||
tokgen = tokenize.generate_tokens(readline)
|
|
||||||
for toktype, ttext, (slineno, scol), (elineno, ecol), ltext in tokgen:
|
|
||||||
if toktype == token.STRING and (prev_toktype == token.INDENT):
|
|
||||||
source_flag = replace_char_between(slineno, scol, elineno, ecol, source_flag, ' ', ls)
|
|
||||||
elif toktype == token.STRING and (prev_toktype == token.NEWLINE):
|
|
||||||
source_flag = replace_char_between(slineno, scol, elineno, ecol, source_flag, ' ', ls)
|
|
||||||
elif toktype == tokenize.COMMENT:
|
|
||||||
source_flag = replace_char_between(slineno, scol, elineno, ecol, source_flag, ' ', ls)
|
|
||||||
prev_toktype = toktype
|
|
||||||
return source_flag
|
|
||||||
|
|
||||||
|
|
||||||
# 示例使用
|
# 示例使用
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -1,450 +0,0 @@
|
|||||||
import os
|
|
||||||
import time
|
|
||||||
from abc import ABC, abstractmethod
|
|
||||||
from datetime import datetime
|
|
||||||
from docx import Document
|
|
||||||
from docx.enum.style import WD_STYLE_TYPE
|
|
||||||
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT, WD_LINE_SPACING
|
|
||||||
from docx.oxml.ns import qn
|
|
||||||
from docx.shared import Inches, Cm
|
|
||||||
from docx.shared import Pt, RGBColor, Inches
|
|
||||||
from typing import Dict, List, Tuple
|
|
||||||
|
|
||||||
|
|
||||||
class DocumentFormatter(ABC):
|
|
||||||
"""文档格式化基类，定义文档格式化的基本接口"""
|
|
||||||
|
|
||||||
def __init__(self, final_summary: str, file_summaries_map: Dict, failed_files: List[Tuple]):
|
|
||||||
self.final_summary = final_summary
|
|
||||||
self.file_summaries_map = file_summaries_map
|
|
||||||
self.failed_files = failed_files
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def format_failed_files(self) -> str:
|
|
||||||
"""格式化失败文件列表"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def format_file_summaries(self) -> str:
|
|
||||||
"""格式化文件总结内容"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
@abstractmethod
|
|
||||||
def create_document(self) -> str:
|
|
||||||
"""创建完整文档"""
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class WordFormatter(DocumentFormatter):
|
|
||||||
"""Word格式文档生成器 - 符合中国政府公文格式规范(GB/T 9704-2012)，并进行了优化"""
|
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
super().__init__(*args, **kwargs)
|
|
||||||
self.doc = Document()
|
|
||||||
self._setup_document()
|
|
||||||
self._create_styles()
|
|
||||||
# 初始化三级标题编号系统
|
|
||||||
self.numbers = {
|
|
||||||
1: 0, # 一级标题编号
|
|
||||||
2: 0, # 二级标题编号
|
|
||||||
3: 0 # 三级标题编号
|
|
||||||
}
|
|
||||||
|
|
||||||
def _setup_document(self):
|
|
||||||
"""设置文档基本格式，包括页面设置和页眉"""
|
|
||||||
sections = self.doc.sections
|
|
||||||
for section in sections:
|
|
||||||
# 设置页面大小为A4
|
|
||||||
section.page_width = Cm(21)
|
|
||||||
section.page_height = Cm(29.7)
|
|
||||||
# 设置页边距
|
|
||||||
section.top_margin = Cm(3.7) # 上边距37mm
|
|
||||||
section.bottom_margin = Cm(3.5) # 下边距35mm
|
|
||||||
section.left_margin = Cm(2.8) # 左边距28mm
|
|
||||||
section.right_margin = Cm(2.6) # 右边距26mm
|
|
||||||
# 设置页眉页脚距离
|
|
||||||
section.header_distance = Cm(2.0)
|
|
||||||
section.footer_distance = Cm(2.0)
|
|
||||||
|
|
||||||
# 添加页眉
|
|
||||||
header = section.header
|
|
||||||
header_para = header.paragraphs[0]
|
|
||||||
header_para.alignment = WD_PARAGRAPH_ALIGNMENT.RIGHT
|
|
||||||
header_run = header_para.add_run("该文档由GPT-academic生成")
|
|
||||||
header_run.font.name = '仿宋'
|
|
||||||
header_run._element.rPr.rFonts.set(qn('w:eastAsia'), '仿宋')
|
|
||||||
header_run.font.size = Pt(9)
|
|
||||||
|
|
||||||
def _create_styles(self):
|
|
||||||
"""创建文档样式"""
|
|
||||||
# 创建正文样式
|
|
||||||
style = self.doc.styles.add_style('Normal_Custom', WD_STYLE_TYPE.PARAGRAPH)
|
|
||||||
style.font.name = '仿宋'
|
|
||||||
style._element.rPr.rFonts.set(qn('w:eastAsia'), '仿宋')
|
|
||||||
style.font.size = Pt(14)
|
|
||||||
style.paragraph_format.line_spacing_rule = WD_LINE_SPACING.ONE_POINT_FIVE
|
|
||||||
style.paragraph_format.space_after = Pt(0)
|
|
||||||
style.paragraph_format.first_line_indent = Pt(28)
|
|
||||||
|
|
||||||
# 创建各级标题样式
|
|
||||||
self._create_heading_style('Title_Custom', '方正小标宋简体', 32, WD_PARAGRAPH_ALIGNMENT.CENTER)
|
|
||||||
self._create_heading_style('Heading1_Custom', '黑体', 22, WD_PARAGRAPH_ALIGNMENT.LEFT)
|
|
||||||
self._create_heading_style('Heading2_Custom', '黑体', 18, WD_PARAGRAPH_ALIGNMENT.LEFT)
|
|
||||||
self._create_heading_style('Heading3_Custom', '黑体', 16, WD_PARAGRAPH_ALIGNMENT.LEFT)
|
|
||||||
|
|
||||||
def _create_heading_style(self, style_name: str, font_name: str, font_size: int, alignment):
|
|
||||||
"""创建标题样式"""
|
|
||||||
style = self.doc.styles.add_style(style_name, WD_STYLE_TYPE.PARAGRAPH)
|
|
||||||
style.font.name = font_name
|
|
||||||
style._element.rPr.rFonts.set(qn('w:eastAsia'), font_name)
|
|
||||||
style.font.size = Pt(font_size)
|
|
||||||
style.font.bold = True
|
|
||||||
style.paragraph_format.alignment = alignment
|
|
||||||
style.paragraph_format.space_before = Pt(12)
|
|
||||||
style.paragraph_format.space_after = Pt(12)
|
|
||||||
style.paragraph_format.line_spacing_rule = WD_LINE_SPACING.ONE_POINT_FIVE
|
|
||||||
return style
|
|
||||||
|
|
||||||
def _get_heading_number(self, level: int) -> str:
|
|
||||||
"""
|
|
||||||
生成标题编号
|
|
||||||
|
|
||||||
Args:
|
|
||||||
level: 标题级别 (0-3)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: 格式化的标题编号
|
|
||||||
"""
|
|
||||||
if level == 0: # 主标题不需要编号
|
|
||||||
return ""
|
|
||||||
|
|
||||||
self.numbers[level] += 1 # 增加当前级别的编号
|
|
||||||
|
|
||||||
# 重置下级标题编号
|
|
||||||
for i in range(level + 1, 4):
|
|
||||||
self.numbers[i] = 0
|
|
||||||
|
|
||||||
# 根据级别返回不同格式的编号
|
|
||||||
if level == 1:
|
|
||||||
return f"{self.numbers[1]}. "
|
|
||||||
elif level == 2:
|
|
||||||
return f"{self.numbers[1]}.{self.numbers[2]} "
|
|
||||||
elif level == 3:
|
|
||||||
return f"{self.numbers[1]}.{self.numbers[2]}.{self.numbers[3]} "
|
|
||||||
return ""
|
|
||||||
|
|
||||||
def _add_heading(self, text: str, level: int):
|
|
||||||
"""
|
|
||||||
添加带编号的标题
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: 标题文本
|
|
||||||
level: 标题级别 (0-3)
|
|
||||||
"""
|
|
||||||
style_map = {
|
|
||||||
0: 'Title_Custom',
|
|
||||||
1: 'Heading1_Custom',
|
|
||||||
2: 'Heading2_Custom',
|
|
||||||
3: 'Heading3_Custom'
|
|
||||||
}
|
|
||||||
|
|
||||||
number = self._get_heading_number(level)
|
|
||||||
paragraph = self.doc.add_paragraph(style=style_map[level])
|
|
||||||
|
|
||||||
if number:
|
|
||||||
number_run = paragraph.add_run(number)
|
|
||||||
font_size = 22 if level == 1 else (18 if level == 2 else 16)
|
|
||||||
self._get_run_style(number_run, '黑体', font_size, True)
|
|
||||||
|
|
||||||
text_run = paragraph.add_run(text)
|
|
||||||
font_size = 32 if level == 0 else (22 if level == 1 else (18 if level == 2 else 16))
|
|
||||||
self._get_run_style(text_run, '黑体', font_size, True)
|
|
||||||
|
|
||||||
# 主标题添加日期
|
|
||||||
if level == 0:
|
|
||||||
date_paragraph = self.doc.add_paragraph()
|
|
||||||
date_paragraph.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
|
|
||||||
date_run = date_paragraph.add_run(datetime.now().strftime('%Y年%m月%d日'))
|
|
||||||
self._get_run_style(date_run, '仿宋', 16, False)
|
|
||||||
|
|
||||||
return paragraph
|
|
||||||
|
|
||||||
def _get_run_style(self, run, font_name: str, font_size: int, bold: bool = False):
|
|
||||||
"""设置文本运行对象的样式"""
|
|
||||||
run.font.name = font_name
|
|
||||||
run._element.rPr.rFonts.set(qn('w:eastAsia'), font_name)
|
|
||||||
run.font.size = Pt(font_size)
|
|
||||||
run.font.bold = bold
|
|
||||||
|
|
||||||
def format_failed_files(self) -> str:
|
|
||||||
"""格式化失败文件列表"""
|
|
||||||
result = []
|
|
||||||
if not self.failed_files:
|
|
||||||
return "\n".join(result)
|
|
||||||
|
|
||||||
result.append("处理失败文件:")
|
|
||||||
for fp, reason in self.failed_files:
|
|
||||||
result.append(f"• {os.path.basename(fp)}: {reason}")
|
|
||||||
|
|
||||||
self._add_heading("处理失败文件", 1)
|
|
||||||
for fp, reason in self.failed_files:
|
|
||||||
self._add_content(f"• {os.path.basename(fp)}: {reason}", indent=False)
|
|
||||||
self.doc.add_paragraph()
|
|
||||||
|
|
||||||
return "\n".join(result)
|
|
||||||
|
|
||||||
def _add_content(self, text: str, indent: bool = True):
|
|
||||||
"""添加正文内容"""
|
|
||||||
paragraph = self.doc.add_paragraph(text, style='Normal_Custom')
|
|
||||||
if not indent:
|
|
||||||
paragraph.paragraph_format.first_line_indent = Pt(0)
|
|
||||||
return paragraph
|
|
||||||
|
|
||||||
def format_file_summaries(self) -> str:
|
|
||||||
"""
|
|
||||||
格式化文件总结内容，确保正确的标题层级
|
|
||||||
|
|
||||||
返回:
|
|
||||||
str: 格式化后的文件总结字符串
|
|
||||||
|
|
||||||
标题层级规则:
|
|
||||||
1. 一级标题为"各文件详细总结"
|
|
||||||
2. 如果文件有目录路径:
|
|
||||||
- 目录路径作为二级标题 (2.1, 2.2 等)
|
|
||||||
- 该目录下所有文件作为三级标题 (2.1.1, 2.1.2 等)
|
|
||||||
3. 如果文件没有目录路径:
|
|
||||||
- 文件直接作为二级标题 (2.1, 2.2 等)
|
|
||||||
"""
|
|
||||||
result = []
|
|
||||||
# 首先对文件路径进行分组整理
|
|
||||||
file_groups = {}
|
|
||||||
for path in sorted(self.file_summaries_map.keys()):
|
|
||||||
dir_path = os.path.dirname(path)
|
|
||||||
if dir_path not in file_groups:
|
|
||||||
file_groups[dir_path] = []
|
|
||||||
file_groups[dir_path].append(path)
|
|
||||||
|
|
||||||
# 处理没有目录的文件
|
|
||||||
root_files = file_groups.get("", [])
|
|
||||||
if root_files:
|
|
||||||
for path in sorted(root_files):
|
|
||||||
file_name = os.path.basename(path)
|
|
||||||
result.append(f"\n📄 {file_name}")
|
|
||||||
result.append(self.file_summaries_map[path])
|
|
||||||
# 无目录的文件作为二级标题
|
|
||||||
self._add_heading(f"📄 {file_name}", 2)
|
|
||||||
self._add_content(self.file_summaries_map[path])
|
|
||||||
self.doc.add_paragraph()
|
|
||||||
|
|
||||||
# 处理有目录的文件
|
|
||||||
for dir_path in sorted(file_groups.keys()):
|
|
||||||
if dir_path == "": # 跳过已处理的根目录文件
|
|
||||||
continue
|
|
||||||
|
|
||||||
# 添加目录作为二级标题
|
|
||||||
result.append(f"\n📁 {dir_path}")
|
|
||||||
self._add_heading(f"📁 {dir_path}", 2)
|
|
||||||
|
|
||||||
# 该目录下的所有文件作为三级标题
|
|
||||||
for path in sorted(file_groups[dir_path]):
|
|
||||||
file_name = os.path.basename(path)
|
|
||||||
result.append(f"\n📄 {file_name}")
|
|
||||||
result.append(self.file_summaries_map[path])
|
|
||||||
|
|
||||||
# 添加文件名作为三级标题
|
|
||||||
self._add_heading(f"📄 {file_name}", 3)
|
|
||||||
self._add_content(self.file_summaries_map[path])
|
|
||||||
self.doc.add_paragraph()
|
|
||||||
|
|
||||||
return "\n".join(result)
|
|
||||||
|
|
||||||
|
|
||||||
def create_document(self):
|
|
||||||
"""创建完整Word文档并返回文档对象"""
|
|
||||||
# 重置所有编号
|
|
||||||
for level in self.numbers:
|
|
||||||
self.numbers[level] = 0
|
|
||||||
|
|
||||||
# 添加主标题
|
|
||||||
self._add_heading("文档总结报告", 0)
|
|
||||||
self.doc.add_paragraph()
|
|
||||||
|
|
||||||
# 添加总体摘要
|
|
||||||
self._add_heading("总体摘要", 1)
|
|
||||||
self._add_content(self.final_summary)
|
|
||||||
self.doc.add_paragraph()
|
|
||||||
|
|
||||||
# 添加失败文件列表（如果有）
|
|
||||||
if self.failed_files:
|
|
||||||
self.format_failed_files()
|
|
||||||
|
|
||||||
# 添加文件详细总结
|
|
||||||
self._add_heading("各文件详细总结", 1)
|
|
||||||
self.format_file_summaries()
|
|
||||||
|
|
||||||
return self.doc
|
|
||||||
|
|
||||||
|
|
||||||
class MarkdownFormatter(DocumentFormatter):
|
|
||||||
"""Markdown格式文档生成器"""
|
|
||||||
|
|
||||||
def format_failed_files(self) -> str:
|
|
||||||
if not self.failed_files:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
formatted_text = ["\n## ⚠️ 处理失败的文件"]
|
|
||||||
for fp, reason in self.failed_files:
|
|
||||||
formatted_text.append(f"- {os.path.basename(fp)}: {reason}")
|
|
||||||
formatted_text.append("\n---")
|
|
||||||
return "\n".join(formatted_text)
|
|
||||||
|
|
||||||
def format_file_summaries(self) -> str:
|
|
||||||
formatted_text = []
|
|
||||||
sorted_paths = sorted(self.file_summaries_map.keys())
|
|
||||||
current_dir = ""
|
|
||||||
|
|
||||||
for path in sorted_paths:
|
|
||||||
dir_path = os.path.dirname(path)
|
|
||||||
if dir_path != current_dir:
|
|
||||||
if dir_path:
|
|
||||||
formatted_text.append(f"\n## 📁 {dir_path}")
|
|
||||||
current_dir = dir_path
|
|
||||||
|
|
||||||
file_name = os.path.basename(path)
|
|
||||||
formatted_text.append(f"\n### 📄 {file_name}")
|
|
||||||
formatted_text.append(self.file_summaries_map[path])
|
|
||||||
formatted_text.append("\n---")
|
|
||||||
|
|
||||||
return "\n".join(formatted_text)
|
|
||||||
|
|
||||||
def create_document(self) -> str:
|
|
||||||
document = [
|
|
||||||
"# 📑 文档总结报告",
|
|
||||||
"\n## 总体摘要",
|
|
||||||
self.final_summary
|
|
||||||
]
|
|
||||||
|
|
||||||
if self.failed_files:
|
|
||||||
document.append(self.format_failed_files())
|
|
||||||
|
|
||||||
document.extend([
|
|
||||||
"\n# 📚 各文件详细总结",
|
|
||||||
self.format_file_summaries()
|
|
||||||
])
|
|
||||||
|
|
||||||
return "\n".join(document)
|
|
||||||
|
|
||||||
|
|
||||||
class HtmlFormatter(DocumentFormatter):
|
|
||||||
"""HTML格式文档生成器"""
|
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
super().__init__(*args, **kwargs)
|
|
||||||
self.css_styles = """
|
|
||||||
body {
|
|
||||||
font-family: "Microsoft YaHei", Arial, sans-serif;
|
|
||||||
line-height: 1.6;
|
|
||||||
max-width: 1000px;
|
|
||||||
margin: 0 auto;
|
|
||||||
padding: 20px;
|
|
||||||
color: #333;
|
|
||||||
}
|
|
||||||
h1 {
|
|
||||||
color: #2c3e50;
|
|
||||||
border-bottom: 2px solid #eee;
|
|
||||||
padding-bottom: 10px;
|
|
||||||
font-size: 24px;
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
h2 {
|
|
||||||
color: #34495e;
|
|
||||||
margin-top: 30px;
|
|
||||||
font-size: 20px;
|
|
||||||
border-left: 4px solid #3498db;
|
|
||||||
padding-left: 10px;
|
|
||||||
}
|
|
||||||
h3 {
|
|
||||||
color: #2c3e50;
|
|
||||||
font-size: 18px;
|
|
||||||
margin-top: 20px;
|
|
||||||
}
|
|
||||||
.summary {
|
|
||||||
background-color: #f8f9fa;
|
|
||||||
padding: 20px;
|
|
||||||
border-radius: 5px;
|
|
||||||
margin: 20px 0;
|
|
||||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
|
||||||
}
|
|
||||||
.details {
|
|
||||||
margin-top: 40px;
|
|
||||||
}
|
|
||||||
.failed-files {
|
|
||||||
background-color: #fff3f3;
|
|
||||||
padding: 15px;
|
|
||||||
border-left: 4px solid #e74c3c;
|
|
||||||
margin: 20px 0;
|
|
||||||
}
|
|
||||||
.file-summary {
|
|
||||||
background-color: #fff;
|
|
||||||
padding: 15px;
|
|
||||||
margin: 15px 0;
|
|
||||||
border-radius: 4px;
|
|
||||||
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
def format_failed_files(self) -> str:
|
|
||||||
if not self.failed_files:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
failed_files_html = ['<div class="failed-files">']
|
|
||||||
failed_files_html.append("<h2>⚠️ 处理失败的文件</h2>")
|
|
||||||
failed_files_html.append("<ul>")
|
|
||||||
for fp, reason in self.failed_files:
|
|
||||||
failed_files_html.append(f"<li><strong>{os.path.basename(fp)}:</strong> {reason}</li>")
|
|
||||||
failed_files_html.append("</ul></div>")
|
|
||||||
return "\n".join(failed_files_html)
|
|
||||||
|
|
||||||
def format_file_summaries(self) -> str:
|
|
||||||
formatted_html = []
|
|
||||||
sorted_paths = sorted(self.file_summaries_map.keys())
|
|
||||||
current_dir = ""
|
|
||||||
|
|
||||||
for path in sorted_paths:
|
|
||||||
dir_path = os.path.dirname(path)
|
|
||||||
if dir_path != current_dir:
|
|
||||||
if dir_path:
|
|
||||||
formatted_html.append(f'<h2>📁 {dir_path}</h2>')
|
|
||||||
current_dir = dir_path
|
|
||||||
|
|
||||||
file_name = os.path.basename(path)
|
|
||||||
formatted_html.append('<div class="file-summary">')
|
|
||||||
formatted_html.append(f'<h3>📄 {file_name}</h3>')
|
|
||||||
formatted_html.append(f'<p>{self.file_summaries_map[path]}</p>')
|
|
||||||
formatted_html.append('</div>')
|
|
||||||
|
|
||||||
return "\n".join(formatted_html)
|
|
||||||
|
|
||||||
def create_document(self) -> str:
|
|
||||||
return f"""
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<meta charset='utf-8'>
|
|
||||||
<title>文档总结报告</title>
|
|
||||||
<style>{self.css_styles}</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>📑 文档总结报告</h1>
|
|
||||||
<h2>总体摘要</h2>
|
|
||||||
<div class="summary">{self.final_summary}</div>
|
|
||||||
{self.format_failed_files()}
|
|
||||||
<div class="details">
|
|
||||||
<h2>📚 各文件详细总结</h2>
|
|
||||||
{self.format_file_summaries()}
|
|
||||||
</div>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
@@ -3,7 +3,7 @@ import re
|
|||||||
import shutil
|
import shutil
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from toolbox import update_ui, update_ui_lastest_msg, get_log_folder, gen_time_str
|
from toolbox import update_ui, update_ui_lastest_msg, get_log_folder
|
||||||
from toolbox import get_conf, promote_file_to_downloadzone
|
from toolbox import get_conf, promote_file_to_downloadzone
|
||||||
from crazy_functions.latex_fns.latex_toolbox import PRESERVE, TRANSFORM
|
from crazy_functions.latex_fns.latex_toolbox import PRESERVE, TRANSFORM
|
||||||
from crazy_functions.latex_fns.latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace
|
from crazy_functions.latex_fns.latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace
|
||||||
@@ -468,70 +468,3 @@ def write_html(sp_file_contents, sp_file_result, chatbot, project_folder):
|
|||||||
except:
|
except:
|
||||||
from toolbox import trimmed_format_exc
|
from toolbox import trimmed_format_exc
|
||||||
logger.error('writing html result failed:', trimmed_format_exc())
|
logger.error('writing html result failed:', trimmed_format_exc())
|
||||||
|
|
||||||
|
|
||||||
def upload_to_gptac_cloud_if_user_allow(chatbot, arxiv_id):
|
|
||||||
try:
|
|
||||||
# 如果用户允许，我们将arxiv论文PDF上传到GPTAC学术云
|
|
||||||
from toolbox import map_file_to_sha256
|
|
||||||
# 检查是否顺利，如果没有生成预期的文件，则跳过
|
|
||||||
is_result_good = False
|
|
||||||
for file_path in chatbot._cookies.get("files_to_promote", []):
|
|
||||||
if file_path.endswith('translate_zh.pdf'):
|
|
||||||
is_result_good = True
|
|
||||||
if not is_result_good:
|
|
||||||
return
|
|
||||||
# 上传文件
|
|
||||||
for file_path in chatbot._cookies.get("files_to_promote", []):
|
|
||||||
align_name = None
|
|
||||||
# normalized name
|
|
||||||
for name in ['translate_zh.pdf', 'comparison.pdf']:
|
|
||||||
if file_path.endswith(name): align_name = name
|
|
||||||
# if match any align name
|
|
||||||
if align_name:
|
|
||||||
logger.info(f'Uploading to GPTAC cloud as the user has set `allow_cloud_io`: {file_path}')
|
|
||||||
with open(file_path, 'rb') as f:
|
|
||||||
import requests
|
|
||||||
url = 'https://cloud-2.agent-matrix.com/arxiv_tf_paper_normal_upload'
|
|
||||||
files = {'file': (align_name, f, 'application/octet-stream')}
|
|
||||||
data = {
|
|
||||||
'arxiv_id': arxiv_id,
|
|
||||||
'file_hash': map_file_to_sha256(file_path),
|
|
||||||
'language': 'zh',
|
|
||||||
'trans_prompt': 'to_be_implemented',
|
|
||||||
'llm_model': 'to_be_implemented',
|
|
||||||
'llm_model_param': 'to_be_implemented',
|
|
||||||
}
|
|
||||||
resp = requests.post(url=url, files=files, data=data, timeout=30)
|
|
||||||
logger.info(f'Uploading terminate ({resp.status_code})`: {file_path}')
|
|
||||||
except:
|
|
||||||
# 如果上传失败，不会中断程序，因为这是次要功能
|
|
||||||
pass
|
|
||||||
|
|
||||||
def check_gptac_cloud(arxiv_id, chatbot):
|
|
||||||
import requests
|
|
||||||
success = False
|
|
||||||
downloaded = []
|
|
||||||
try:
|
|
||||||
for pdf_target in ['translate_zh.pdf', 'comparison.pdf']:
|
|
||||||
url = 'https://cloud-2.agent-matrix.com/arxiv_tf_paper_normal_exist'
|
|
||||||
data = {
|
|
||||||
'arxiv_id': arxiv_id,
|
|
||||||
'name': pdf_target,
|
|
||||||
}
|
|
||||||
resp = requests.post(url=url, data=data)
|
|
||||||
cache_hit_result = resp.text.strip('"')
|
|
||||||
if cache_hit_result.startswith("http"):
|
|
||||||
url = cache_hit_result
|
|
||||||
logger.info(f'Downloading from GPTAC cloud: {url}')
|
|
||||||
resp = requests.get(url=url, timeout=30)
|
|
||||||
target = os.path.join(get_log_folder(plugin_name='gptac_cloud'), gen_time_str(), pdf_target)
|
|
||||||
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
||||||
with open(target, 'wb') as f:
|
|
||||||
f.write(resp.content)
|
|
||||||
new_path = promote_file_to_downloadzone(target, chatbot=chatbot)
|
|
||||||
success = True
|
|
||||||
downloaded.append(new_path)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
return success, downloaded
|
|
||||||
|
|||||||
@@ -4,9 +4,7 @@ from toolbox import promote_file_to_downloadzone, extract_archive
|
|||||||
from toolbox import generate_file_link, zip_folder
|
from toolbox import generate_file_link, zip_folder
|
||||||
from crazy_functions.crazy_utils import get_files_from_everything
|
from crazy_functions.crazy_utils import get_files_from_everything
|
||||||
from shared_utils.colorful import *
|
from shared_utils.colorful import *
|
||||||
from loguru import logger
|
|
||||||
import os
|
import os
|
||||||
import time
|
|
||||||
|
|
||||||
def refresh_key(doc2x_api_key):
|
def refresh_key(doc2x_api_key):
|
||||||
import requests, json
|
import requests, json
|
||||||
@@ -24,140 +22,105 @@ def refresh_key(doc2x_api_key):
|
|||||||
raise RuntimeError(format("[ERROR] status code: %d, body: %s" % (res.status_code, res.text)))
|
raise RuntimeError(format("[ERROR] status code: %d, body: %s" % (res.status_code, res.text)))
|
||||||
return doc2x_api_key
|
return doc2x_api_key
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def 解析PDF_DOC2X_转Latex(pdf_file_path):
|
def 解析PDF_DOC2X_转Latex(pdf_file_path):
|
||||||
zip_file_path, unzipped_folder = 解析PDF_DOC2X(pdf_file_path, format='tex')
|
|
||||||
return unzipped_folder
|
|
||||||
|
|
||||||
|
|
||||||
def 解析PDF_DOC2X(pdf_file_path, format='tex'):
|
|
||||||
"""
|
|
||||||
format: 'tex', 'md', 'docx'
|
|
||||||
"""
|
|
||||||
import requests, json, os
|
import requests, json, os
|
||||||
DOC2X_API_KEY = get_conf('DOC2X_API_KEY')
|
DOC2X_API_KEY = get_conf('DOC2X_API_KEY')
|
||||||
latex_dir = get_log_folder(plugin_name="pdf_ocr_latex")
|
latex_dir = get_log_folder(plugin_name="pdf_ocr_latex")
|
||||||
markdown_dir = get_log_folder(plugin_name="pdf_ocr")
|
|
||||||
doc2x_api_key = DOC2X_API_KEY
|
doc2x_api_key = DOC2X_API_KEY
|
||||||
|
if doc2x_api_key.startswith('sk-'):
|
||||||
|
url = "https://api.doc2x.noedgeai.com/api/v1/pdf"
|
||||||
# < ------ 第1步：上传 ------ >
|
|
||||||
logger.info("Doc2x 第1步：上传")
|
|
||||||
with open(pdf_file_path, 'rb') as file:
|
|
||||||
res = requests.post(
|
|
||||||
"https://v2.doc2x.noedgeai.com/api/v2/parse/pdf",
|
|
||||||
headers={"Authorization": "Bearer " + doc2x_api_key},
|
|
||||||
data=file
|
|
||||||
)
|
|
||||||
# res_json = []
|
|
||||||
if res.status_code == 200:
|
|
||||||
res_json = res.json()
|
|
||||||
else:
|
else:
|
||||||
raise RuntimeError(f"Doc2x return an error: {res.json()}")
|
doc2x_api_key = refresh_key(doc2x_api_key)
|
||||||
uuid = res_json['data']['uid']
|
url = "https://api.doc2x.noedgeai.com/api/platform/pdf"
|
||||||
|
|
||||||
# < ------ 第2步：轮询等待 ------ >
|
|
||||||
logger.info("Doc2x 第2步：轮询等待")
|
|
||||||
params = {'uid': uuid}
|
|
||||||
while True:
|
|
||||||
res = requests.get(
|
|
||||||
'https://v2.doc2x.noedgeai.com/api/v2/parse/status',
|
|
||||||
headers={"Authorization": "Bearer " + doc2x_api_key},
|
|
||||||
params=params
|
|
||||||
)
|
|
||||||
res_json = res.json()
|
|
||||||
if res_json['data']['status'] == "success":
|
|
||||||
break
|
|
||||||
elif res_json['data']['status'] == "processing":
|
|
||||||
time.sleep(3)
|
|
||||||
logger.info(f"Doc2x is processing at {res_json['data']['progress']}%")
|
|
||||||
elif res_json['data']['status'] == "failed":
|
|
||||||
raise RuntimeError(f"Doc2x return an error: {res_json}")
|
|
||||||
|
|
||||||
|
|
||||||
# < ------ 第3步：提交转化 ------ >
|
|
||||||
logger.info("Doc2x 第3步：提交转化")
|
|
||||||
data = {
|
|
||||||
"uid": uuid,
|
|
||||||
"to": format,
|
|
||||||
"formula_mode": "dollar",
|
|
||||||
"filename": "output"
|
|
||||||
}
|
|
||||||
res = requests.post(
|
res = requests.post(
|
||||||
'https://v2.doc2x.noedgeai.com/api/v2/convert/parse',
|
url,
|
||||||
headers={"Authorization": "Bearer " + doc2x_api_key},
|
files={"file": open(pdf_file_path, "rb")},
|
||||||
json=data
|
data={"ocr": "1"},
|
||||||
|
headers={"Authorization": "Bearer " + doc2x_api_key}
|
||||||
)
|
)
|
||||||
|
res_json = []
|
||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
res_json = res.json()
|
decoded = res.content.decode("utf-8")
|
||||||
|
for z_decoded in decoded.split('\n'):
|
||||||
|
if len(z_decoded) == 0: continue
|
||||||
|
assert z_decoded.startswith("data: ")
|
||||||
|
z_decoded = z_decoded[len("data: "):]
|
||||||
|
decoded_json = json.loads(z_decoded)
|
||||||
|
res_json.append(decoded_json)
|
||||||
else:
|
else:
|
||||||
raise RuntimeError(f"Doc2x return an error: {res.json()}")
|
raise RuntimeError(format("[ERROR] status code: %d, body: %s" % (res.status_code, res.text)))
|
||||||
|
|
||||||
|
uuid = res_json[0]['uuid']
|
||||||
|
to = "latex" # latex, md, docx
|
||||||
|
url = "https://api.doc2x.noedgeai.com/api/export"+"?request_id="+uuid+"&to="+to
|
||||||
|
|
||||||
# < ------ 第4步：等待结果 ------ >
|
res = requests.get(url, headers={"Authorization": "Bearer " + doc2x_api_key})
|
||||||
logger.info("Doc2x 第4步：等待结果")
|
latex_zip_path = os.path.join(latex_dir, gen_time_str() + '.zip')
|
||||||
params = {'uid': uuid}
|
latex_unzip_path = os.path.join(latex_dir, gen_time_str())
|
||||||
while True:
|
if res.status_code == 200:
|
||||||
res = requests.get(
|
with open(latex_zip_path, "wb") as f: f.write(res.content)
|
||||||
'https://v2.doc2x.noedgeai.com/api/v2/convert/parse/result',
|
else:
|
||||||
headers={"Authorization": "Bearer " + doc2x_api_key},
|
raise RuntimeError(format("[ERROR] status code: %d, body: %s" % (res.status_code, res.text)))
|
||||||
params=params
|
|
||||||
)
|
|
||||||
res_json = res.json()
|
|
||||||
if res_json['data']['status'] == "success":
|
|
||||||
break
|
|
||||||
elif res_json['data']['status'] == "processing":
|
|
||||||
time.sleep(3)
|
|
||||||
logger.info(f"Doc2x still processing")
|
|
||||||
elif res_json['data']['status'] == "failed":
|
|
||||||
raise RuntimeError(f"Doc2x return an error: {res_json}")
|
|
||||||
|
|
||||||
|
|
||||||
# < ------ 第5步：最后的处理 ------ >
|
|
||||||
logger.info("Doc2x 第5步：最后的处理")
|
|
||||||
|
|
||||||
if format=='tex':
|
|
||||||
target_path = latex_dir
|
|
||||||
if format=='md':
|
|
||||||
target_path = markdown_dir
|
|
||||||
os.makedirs(target_path, exist_ok=True)
|
|
||||||
|
|
||||||
max_attempt = 3
|
|
||||||
# < ------ 下载 ------ >
|
|
||||||
for attempt in range(max_attempt):
|
|
||||||
try:
|
|
||||||
result_url = res_json['data']['url']
|
|
||||||
res = requests.get(result_url)
|
|
||||||
zip_path = os.path.join(target_path, gen_time_str() + '.zip')
|
|
||||||
unzip_path = os.path.join(target_path, gen_time_str())
|
|
||||||
if res.status_code == 200:
|
|
||||||
with open(zip_path, "wb") as f: f.write(res.content)
|
|
||||||
else:
|
|
||||||
raise RuntimeError(f"Doc2x return an error: {res.json()}")
|
|
||||||
except Exception as e:
|
|
||||||
if attempt < max_attempt - 1:
|
|
||||||
logger.error(f"Failed to download latex file, retrying... {e}")
|
|
||||||
time.sleep(3)
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
raise e
|
|
||||||
|
|
||||||
# < ------ 解压 ------ >
|
|
||||||
import zipfile
|
import zipfile
|
||||||
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
|
with zipfile.ZipFile(latex_zip_path, 'r') as zip_ref:
|
||||||
zip_ref.extractall(unzip_path)
|
zip_ref.extractall(latex_unzip_path)
|
||||||
return zip_path, unzip_path
|
|
||||||
|
|
||||||
|
return latex_unzip_path
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def 解析PDF_DOC2X_单文件(fp, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, DOC2X_API_KEY, user_request):
|
def 解析PDF_DOC2X_单文件(fp, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, DOC2X_API_KEY, user_request):
|
||||||
|
|
||||||
|
|
||||||
def pdf2markdown(filepath):
|
def pdf2markdown(filepath):
|
||||||
chatbot.append((None, f"Doc2x 解析中"))
|
import requests, json, os
|
||||||
|
markdown_dir = get_log_folder(plugin_name="pdf_ocr")
|
||||||
|
doc2x_api_key = DOC2X_API_KEY
|
||||||
|
if doc2x_api_key.startswith('sk-'):
|
||||||
|
url = "https://api.doc2x.noedgeai.com/api/v1/pdf"
|
||||||
|
else:
|
||||||
|
doc2x_api_key = refresh_key(doc2x_api_key)
|
||||||
|
url = "https://api.doc2x.noedgeai.com/api/platform/pdf"
|
||||||
|
|
||||||
|
chatbot.append((None, "加载PDF文件，发送至DOC2X解析..."))
|
||||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
|
||||||
md_zip_path, unzipped_folder = 解析PDF_DOC2X(filepath, format='md')
|
res = requests.post(
|
||||||
|
url,
|
||||||
|
files={"file": open(filepath, "rb")},
|
||||||
|
data={"ocr": "1"},
|
||||||
|
headers={"Authorization": "Bearer " + doc2x_api_key}
|
||||||
|
)
|
||||||
|
res_json = []
|
||||||
|
if res.status_code == 200:
|
||||||
|
decoded = res.content.decode("utf-8")
|
||||||
|
for z_decoded in decoded.split('\n'):
|
||||||
|
if len(z_decoded) == 0: continue
|
||||||
|
assert z_decoded.startswith("data: ")
|
||||||
|
z_decoded = z_decoded[len("data: "):]
|
||||||
|
decoded_json = json.loads(z_decoded)
|
||||||
|
res_json.append(decoded_json)
|
||||||
|
if 'limit exceeded' in decoded_json.get('status', ''):
|
||||||
|
raise RuntimeError("Doc2x API 页数受限，请联系 Doc2x 方面，并更换新的 API 秘钥。")
|
||||||
|
else:
|
||||||
|
raise RuntimeError(format("[ERROR] status code: %d, body: %s" % (res.status_code, res.text)))
|
||||||
|
uuid = res_json[0]['uuid']
|
||||||
|
to = "md" # latex, md, docx
|
||||||
|
url = "https://api.doc2x.noedgeai.com/api/export"+"?request_id="+uuid+"&to="+to
|
||||||
|
|
||||||
|
chatbot.append((None, f"读取解析: {url} ..."))
|
||||||
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
|
||||||
|
res = requests.get(url, headers={"Authorization": "Bearer " + doc2x_api_key})
|
||||||
|
md_zip_path = os.path.join(markdown_dir, gen_time_str() + '.zip')
|
||||||
|
if res.status_code == 200:
|
||||||
|
with open(md_zip_path, "wb") as f: f.write(res.content)
|
||||||
|
else:
|
||||||
|
raise RuntimeError(format("[ERROR] status code: %d, body: %s" % (res.status_code, res.text)))
|
||||||
promote_file_to_downloadzone(md_zip_path, chatbot=chatbot)
|
promote_file_to_downloadzone(md_zip_path, chatbot=chatbot)
|
||||||
chatbot.append((None, f"完成解析 {md_zip_path} ..."))
|
chatbot.append((None, f"完成解析 {md_zip_path} ..."))
|
||||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
|||||||
@@ -1,13 +1,17 @@
|
|||||||
|
import llama_index
|
||||||
|
import os
|
||||||
import atexit
|
import atexit
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
from llama_index.core import Document
|
from llama_index.core import Document
|
||||||
from llama_index.core.ingestion import run_transformations
|
|
||||||
from llama_index.core.schema import TextNode
|
from llama_index.core.schema import TextNode
|
||||||
|
|
||||||
from crazy_functions.rag_fns.vector_store_index import GptacVectorStoreIndex
|
|
||||||
from request_llms.embed_models.openai_embed import OpenAiEmbeddingModel
|
from request_llms.embed_models.openai_embed import OpenAiEmbeddingModel
|
||||||
|
from shared_utils.connect_void_terminal import get_chat_default_kwargs
|
||||||
|
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
|
||||||
|
from crazy_functions.rag_fns.vector_store_index import GptacVectorStoreIndex
|
||||||
|
from llama_index.core.ingestion import run_transformations
|
||||||
|
from llama_index.core import PromptTemplate
|
||||||
|
from llama_index.core.response_synthesizers import TreeSummarize
|
||||||
|
|
||||||
DEFAULT_QUERY_GENERATION_PROMPT = """\
|
DEFAULT_QUERY_GENERATION_PROMPT = """\
|
||||||
Now, you have context information as below:
|
Now, you have context information as below:
|
||||||
@@ -59,7 +63,7 @@ class SaveLoad():
|
|||||||
def purge(self):
|
def purge(self):
|
||||||
import shutil
|
import shutil
|
||||||
shutil.rmtree(self.checkpoint_dir, ignore_errors=True)
|
shutil.rmtree(self.checkpoint_dir, ignore_errors=True)
|
||||||
self.vs_index = self.create_new_vs(self.checkpoint_dir)
|
self.vs_index = self.create_new_vs()
|
||||||
|
|
||||||
|
|
||||||
class LlamaIndexRagWorker(SaveLoad):
|
class LlamaIndexRagWorker(SaveLoad):
|
||||||
@@ -71,7 +75,7 @@ class LlamaIndexRagWorker(SaveLoad):
|
|||||||
if auto_load_checkpoint:
|
if auto_load_checkpoint:
|
||||||
self.vs_index = self.load_from_checkpoint(checkpoint_dir)
|
self.vs_index = self.load_from_checkpoint(checkpoint_dir)
|
||||||
else:
|
else:
|
||||||
self.vs_index = self.create_new_vs()
|
self.vs_index = self.create_new_vs(checkpoint_dir)
|
||||||
atexit.register(lambda: self.save_to_checkpoint(checkpoint_dir))
|
atexit.register(lambda: self.save_to_checkpoint(checkpoint_dir))
|
||||||
|
|
||||||
def assign_embedding_model(self):
|
def assign_embedding_model(self):
|
||||||
@@ -87,38 +91,32 @@ class LlamaIndexRagWorker(SaveLoad):
|
|||||||
logger.info('oo --------inspect_vector_store end--------')
|
logger.info('oo --------inspect_vector_store end--------')
|
||||||
return vector_store_preview
|
return vector_store_preview
|
||||||
|
|
||||||
def add_documents_to_vector_store(self, document_list: List[Document]):
|
def add_documents_to_vector_store(self, document_list):
|
||||||
"""
|
documents = [Document(text=t) for t in document_list]
|
||||||
Adds a list of Document objects to the vector store after processing.
|
|
||||||
"""
|
|
||||||
documents = document_list
|
|
||||||
documents_nodes = run_transformations(
|
documents_nodes = run_transformations(
|
||||||
documents, # type: ignore
|
documents, # type: ignore
|
||||||
self.vs_index._transformations,
|
self.vs_index._transformations,
|
||||||
show_progress=True
|
show_progress=True
|
||||||
)
|
)
|
||||||
self.vs_index.insert_nodes(documents_nodes)
|
self.vs_index.insert_nodes(documents_nodes)
|
||||||
if self.debug_mode:
|
if self.debug_mode: self.inspect_vector_store()
|
||||||
self.inspect_vector_store()
|
|
||||||
|
|
||||||
def add_text_to_vector_store(self, text: str):
|
def add_text_to_vector_store(self, text):
|
||||||
node = TextNode(text=text)
|
node = TextNode(text=text)
|
||||||
documents_nodes = run_transformations(
|
documents_nodes = run_transformations(
|
||||||
[node],
|
[node],
|
||||||
self.vs_index._transformations,
|
self.vs_index._transformations,
|
||||||
show_progress=True
|
show_progress=True
|
||||||
)
|
)
|
||||||
self.vs_index.insert_nodes(documents_nodes)
|
self.vs_index.insert_nodes(documents_nodes)
|
||||||
if self.debug_mode:
|
if self.debug_mode: self.inspect_vector_store()
|
||||||
self.inspect_vector_store()
|
|
||||||
|
|
||||||
def remember_qa(self, question, answer):
|
def remember_qa(self, question, answer):
|
||||||
formatted_str = QUESTION_ANSWER_RECORD.format(question=question, answer=answer)
|
formatted_str = QUESTION_ANSWER_RECORD.format(question=question, answer=answer)
|
||||||
self.add_text_to_vector_store(formatted_str)
|
self.add_text_to_vector_store(formatted_str)
|
||||||
|
|
||||||
def retrieve_from_store_with_query(self, query):
|
def retrieve_from_store_with_query(self, query):
|
||||||
if self.debug_mode:
|
if self.debug_mode: self.inspect_vector_store()
|
||||||
self.inspect_vector_store()
|
|
||||||
retriever = self.vs_index.as_retriever()
|
retriever = self.vs_index.as_retriever()
|
||||||
return retriever.retrieve(query)
|
return retriever.retrieve(query)
|
||||||
|
|
||||||
@@ -130,9 +128,3 @@ class LlamaIndexRagWorker(SaveLoad):
|
|||||||
buf = "\n".join(([f"(No.{i+1} | score {n.score:.3f}): {n.text}" for i, n in enumerate(nodes)]))
|
buf = "\n".join(([f"(No.{i+1} | score {n.score:.3f}): {n.text}" for i, n in enumerate(nodes)]))
|
||||||
if self.debug_mode: logger.info(buf)
|
if self.debug_mode: logger.info(buf)
|
||||||
return buf
|
return buf
|
||||||
|
|
||||||
def purge_vector_store(self):
|
|
||||||
"""
|
|
||||||
Purges the current vector store and creates a new one.
|
|
||||||
"""
|
|
||||||
self.purge()
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
import os
|
|
||||||
from llama_index.core import SimpleDirectoryReader
|
|
||||||
|
|
||||||
supports_format = ['.csv', '.docx','.doc', '.epub', '.ipynb', '.mbox', '.md', '.pdf', '.txt', '.ppt',
|
|
||||||
'.pptm', '.pptx','.py', '.xls', '.xlsx', '.html', '.json', '.xml', '.yaml', '.yml' ,'.m']
|
|
||||||
|
|
||||||
def read_docx_doc(file_path):
|
|
||||||
if file_path.split(".")[-1] == "docx":
|
|
||||||
from docx import Document
|
|
||||||
doc = Document(file_path)
|
|
||||||
file_content = "\n".join([para.text for para in doc.paragraphs])
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
import win32com.client
|
|
||||||
word = win32com.client.Dispatch("Word.Application")
|
|
||||||
word.visible = False
|
|
||||||
# 打开文件
|
|
||||||
doc = word.Documents.Open(os.getcwd() + '/' + file_path)
|
|
||||||
# file_content = doc.Content.Text
|
|
||||||
doc = word.ActiveDocument
|
|
||||||
file_content = doc.Range().Text
|
|
||||||
doc.Close()
|
|
||||||
word.Quit()
|
|
||||||
except:
|
|
||||||
raise RuntimeError('请先将.doc文档转换为.docx文档。')
|
|
||||||
return file_content
|
|
||||||
|
|
||||||
# 修改后的 extract_text 函数，结合 SimpleDirectoryReader 和自定义解析逻辑
|
|
||||||
import os
|
|
||||||
|
|
||||||
def extract_text(file_path):
|
|
||||||
_, ext = os.path.splitext(file_path.lower())
|
|
||||||
|
|
||||||
# 使用 SimpleDirectoryReader 处理它支持的文件格式
|
|
||||||
if ext in ['.docx', '.doc']:
|
|
||||||
return read_docx_doc(file_path)
|
|
||||||
try:
|
|
||||||
reader = SimpleDirectoryReader(input_files=[file_path])
|
|
||||||
documents = reader.load_data()
|
|
||||||
if len(documents) > 0:
|
|
||||||
return documents[0].text
|
|
||||||
except Exception as e:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return None
|
|
||||||
127
crazy_functions/总结word文档.py
Normal file
127
crazy_functions/总结word文档.py
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
from toolbox import update_ui
|
||||||
|
from toolbox import CatchException, report_exception
|
||||||
|
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||||
|
from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||||
|
fast_debug = False
|
||||||
|
|
||||||
|
|
||||||
|
def 解析docx(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
|
||||||
|
import time, os
|
||||||
|
# pip install python-docx 用于docx格式，跨平台
|
||||||
|
# pip install pywin32 用于doc格式，仅支持Win平台
|
||||||
|
for index, fp in enumerate(file_manifest):
|
||||||
|
if fp.split(".")[-1] == "docx":
|
||||||
|
from docx import Document
|
||||||
|
doc = Document(fp)
|
||||||
|
file_content = "\n".join([para.text for para in doc.paragraphs])
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
import win32com.client
|
||||||
|
word = win32com.client.Dispatch("Word.Application")
|
||||||
|
word.visible = False
|
||||||
|
# 打开文件
|
||||||
|
doc = word.Documents.Open(os.getcwd() + '/' + fp)
|
||||||
|
# file_content = doc.Content.Text
|
||||||
|
doc = word.ActiveDocument
|
||||||
|
file_content = doc.Range().Text
|
||||||
|
doc.Close()
|
||||||
|
word.Quit()
|
||||||
|
except:
|
||||||
|
raise RuntimeError('请先将.doc文档转换为.docx文档。')
|
||||||
|
|
||||||
|
# private_upload里面的文件名在解压zip后容易出现乱码（rar和7z格式正常），故可以只分析文章内容，不输入文件名
|
||||||
|
from crazy_functions.pdf_fns.breakdown_txt import breakdown_text_to_satisfy_token_limit
|
||||||
|
from request_llms.bridge_all import model_info
|
||||||
|
max_token = model_info[llm_kwargs['llm_model']]['max_token']
|
||||||
|
TOKEN_LIMIT_PER_FRAGMENT = max_token * 3 // 4
|
||||||
|
paper_fragments = breakdown_text_to_satisfy_token_limit(txt=file_content, limit=TOKEN_LIMIT_PER_FRAGMENT, llm_model=llm_kwargs['llm_model'])
|
||||||
|
this_paper_history = []
|
||||||
|
for i, paper_frag in enumerate(paper_fragments):
|
||||||
|
i_say = f'请对下面的文章片段用中文做概述，文件名是{os.path.relpath(fp, project_folder)}，文章内容是 ```{paper_frag}```'
|
||||||
|
i_say_show_user = f'请对下面的文章片段做概述: {os.path.abspath(fp)}的第{i+1}/{len(paper_fragments)}个片段。'
|
||||||
|
gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
|
||||||
|
inputs=i_say,
|
||||||
|
inputs_show_user=i_say_show_user,
|
||||||
|
llm_kwargs=llm_kwargs,
|
||||||
|
chatbot=chatbot,
|
||||||
|
history=[],
|
||||||
|
sys_prompt="总结文章。"
|
||||||
|
)
|
||||||
|
|
||||||
|
chatbot[-1] = (i_say_show_user, gpt_say)
|
||||||
|
history.extend([i_say_show_user,gpt_say])
|
||||||
|
this_paper_history.extend([i_say_show_user,gpt_say])
|
||||||
|
|
||||||
|
# 已经对该文章的所有片段总结完毕，如果文章被切分了，
|
||||||
|
if len(paper_fragments) > 1:
|
||||||
|
i_say = f"根据以上的对话，总结文章{os.path.abspath(fp)}的主要内容。"
|
||||||
|
gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
|
||||||
|
inputs=i_say,
|
||||||
|
inputs_show_user=i_say,
|
||||||
|
llm_kwargs=llm_kwargs,
|
||||||
|
chatbot=chatbot,
|
||||||
|
history=this_paper_history,
|
||||||
|
sys_prompt="总结文章。"
|
||||||
|
)
|
||||||
|
|
||||||
|
history.extend([i_say,gpt_say])
|
||||||
|
this_paper_history.extend([i_say,gpt_say])
|
||||||
|
|
||||||
|
res = write_history_to_file(history)
|
||||||
|
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||||
|
chatbot.append(("完成了吗？", res))
|
||||||
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
|
||||||
|
res = write_history_to_file(history)
|
||||||
|
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||||
|
chatbot.append(("所有文件都总结完成了吗？", res))
|
||||||
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
|
||||||
|
|
||||||
|
@CatchException
|
||||||
|
def 总结word文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request):
|
||||||
|
import glob, os
|
||||||
|
|
||||||
|
# 基本信息：功能、贡献者
|
||||||
|
chatbot.append([
|
||||||
|
"函数插件功能？",
|
||||||
|
"批量总结Word文档。函数插件贡献者: JasonGuo1。注意, 如果是.doc文件, 请先转化为.docx格式。"])
|
||||||
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
|
||||||
|
# 尝试导入依赖，如果缺少依赖，则给出安装建议
|
||||||
|
try:
|
||||||
|
from docx import Document
|
||||||
|
except:
|
||||||
|
report_exception(chatbot, history,
|
||||||
|
a=f"解析项目: {txt}",
|
||||||
|
b=f"导入软件依赖失败。使用该模块需要额外依赖，安装方法```pip install --upgrade python-docx pywin32```。")
|
||||||
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
return
|
||||||
|
|
||||||
|
# 清空历史，以免输入溢出
|
||||||
|
history = []
|
||||||
|
|
||||||
|
# 检测输入参数，如没有给定输入参数，直接退出
|
||||||
|
if os.path.exists(txt):
|
||||||
|
project_folder = txt
|
||||||
|
else:
|
||||||
|
if txt == "": txt = '空空如也的输入栏'
|
||||||
|
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
|
||||||
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
return
|
||||||
|
|
||||||
|
# 搜索需要处理的文件清单
|
||||||
|
if txt.endswith('.docx') or txt.endswith('.doc'):
|
||||||
|
file_manifest = [txt]
|
||||||
|
else:
|
||||||
|
file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.docx', recursive=True)] + \
|
||||||
|
[f for f in glob.glob(f'{project_folder}/**/*.doc', recursive=True)]
|
||||||
|
|
||||||
|
# 如果没找到任何文件
|
||||||
|
if len(file_manifest) == 0:
|
||||||
|
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到任何.docx或doc文件: {txt}")
|
||||||
|
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||||
|
return
|
||||||
|
|
||||||
|
# 开始正式执行任务
|
||||||
|
yield from 解析docx(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt)
|
||||||
@@ -1,496 +0,0 @@
|
|||||||
import os
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import List, Tuple, Dict, Generator
|
|
||||||
|
|
||||||
from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
|
|
||||||
from crazy_functions.pdf_fns.breakdown_txt import breakdown_text_to_satisfy_token_limit
|
|
||||||
from crazy_functions.rag_fns.rag_file_support import extract_text
|
|
||||||
from request_llms.bridge_all import model_info
|
|
||||||
from toolbox import update_ui, CatchException, report_exception
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class FileFragment:
|
|
||||||
"""文件片段数据类，用于组织处理单元"""
|
|
||||||
file_path: str
|
|
||||||
content: str
|
|
||||||
rel_path: str
|
|
||||||
fragment_index: int
|
|
||||||
total_fragments: int
|
|
||||||
|
|
||||||
|
|
||||||
class BatchDocumentSummarizer:
|
|
||||||
"""优化的文档总结器 - 批处理版本"""
|
|
||||||
|
|
||||||
def __init__(self, llm_kwargs: Dict, plugin_kwargs: Dict, chatbot: List, history: List, system_prompt: str):
|
|
||||||
"""初始化总结器"""
|
|
||||||
self.llm_kwargs = llm_kwargs
|
|
||||||
self.plugin_kwargs = plugin_kwargs
|
|
||||||
self.chatbot = chatbot
|
|
||||||
self.history = history
|
|
||||||
self.system_prompt = system_prompt
|
|
||||||
self.failed_files = []
|
|
||||||
self.file_summaries_map = {}
|
|
||||||
|
|
||||||
def _get_token_limit(self) -> int:
|
|
||||||
"""获取模型token限制"""
|
|
||||||
max_token = model_info[self.llm_kwargs['llm_model']]['max_token']
|
|
||||||
return max_token * 3 // 4
|
|
||||||
|
|
||||||
def _create_batch_inputs(self, fragments: List[FileFragment]) -> Tuple[List, List, List]:
|
|
||||||
"""创建批处理输入"""
|
|
||||||
inputs_array = []
|
|
||||||
inputs_show_user_array = []
|
|
||||||
history_array = []
|
|
||||||
|
|
||||||
for frag in fragments:
|
|
||||||
if self.plugin_kwargs.get("advanced_arg"):
|
|
||||||
i_say = (f'请按照用户要求对文件内容进行处理，文件名为{os.path.basename(frag.file_path)}，'
|
|
||||||
f'用户要求为：{self.plugin_kwargs["advanced_arg"]}：'
|
|
||||||
f'文件内容是 ```{frag.content}```')
|
|
||||||
i_say_show_user = (f'正在处理 {frag.rel_path} (片段 {frag.fragment_index + 1}/{frag.total_fragments})')
|
|
||||||
else:
|
|
||||||
i_say = (f'请对下面的内容用中文做总结，不超过500字，文件名是{os.path.basename(frag.file_path)}，'
|
|
||||||
f'内容是 ```{frag.content}```')
|
|
||||||
i_say_show_user = f'正在处理 {frag.rel_path} (片段 {frag.fragment_index + 1}/{frag.total_fragments})'
|
|
||||||
|
|
||||||
inputs_array.append(i_say)
|
|
||||||
inputs_show_user_array.append(i_say_show_user)
|
|
||||||
history_array.append([])
|
|
||||||
|
|
||||||
return inputs_array, inputs_show_user_array, history_array
|
|
||||||
|
|
||||||
def _process_single_file_with_timeout(self, file_info: Tuple[str, str], mutable_status: List) -> List[FileFragment]:
|
|
||||||
"""包装了超时控制的文件处理函数"""
|
|
||||||
|
|
||||||
def timeout_handler():
|
|
||||||
thread = threading.current_thread()
|
|
||||||
if hasattr(thread, '_timeout_occurred'):
|
|
||||||
thread._timeout_occurred = True
|
|
||||||
|
|
||||||
# 设置超时标记
|
|
||||||
thread = threading.current_thread()
|
|
||||||
thread._timeout_occurred = False
|
|
||||||
|
|
||||||
# 设置超时定时器
|
|
||||||
timer = threading.Timer(self.watch_dog_patience, timeout_handler)
|
|
||||||
timer.start()
|
|
||||||
|
|
||||||
try:
|
|
||||||
fp, project_folder = file_info
|
|
||||||
fragments = []
|
|
||||||
|
|
||||||
# 定期检查是否超时
|
|
||||||
def check_timeout():
|
|
||||||
if hasattr(thread, '_timeout_occurred') and thread._timeout_occurred:
|
|
||||||
raise TimeoutError("处理超时")
|
|
||||||
|
|
||||||
# 更新状态
|
|
||||||
mutable_status[0] = "检查文件大小"
|
|
||||||
mutable_status[1] = time.time()
|
|
||||||
check_timeout()
|
|
||||||
|
|
||||||
# 文件大小检查
|
|
||||||
if os.path.getsize(fp) > self.max_file_size:
|
|
||||||
self.failed_files.append((fp, f"文件过大：超过{self.max_file_size / 1024 / 1024}MB"))
|
|
||||||
mutable_status[2] = "文件过大"
|
|
||||||
return fragments
|
|
||||||
|
|
||||||
check_timeout()
|
|
||||||
|
|
||||||
# 更新状态
|
|
||||||
mutable_status[0] = "提取文件内容"
|
|
||||||
mutable_status[1] = time.time()
|
|
||||||
|
|
||||||
# 提取内容
|
|
||||||
content = extract_text(fp)
|
|
||||||
if content is None:
|
|
||||||
self.failed_files.append((fp, "文件解析失败：不支持的格式或文件损坏"))
|
|
||||||
mutable_status[2] = "格式不支持"
|
|
||||||
return fragments
|
|
||||||
elif not content.strip():
|
|
||||||
self.failed_files.append((fp, "文件内容为空"))
|
|
||||||
mutable_status[2] = "内容为空"
|
|
||||||
return fragments
|
|
||||||
|
|
||||||
check_timeout()
|
|
||||||
|
|
||||||
# 更新状态
|
|
||||||
mutable_status[0] = "分割文本"
|
|
||||||
mutable_status[1] = time.time()
|
|
||||||
|
|
||||||
# 分割文本
|
|
||||||
try:
|
|
||||||
paper_fragments = breakdown_text_to_satisfy_token_limit(
|
|
||||||
txt=content,
|
|
||||||
limit=self._get_token_limit(),
|
|
||||||
llm_model=self.llm_kwargs['llm_model']
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
self.failed_files.append((fp, f"文本分割失败：{str(e)}"))
|
|
||||||
mutable_status[2] = "分割失败"
|
|
||||||
return fragments
|
|
||||||
|
|
||||||
check_timeout()
|
|
||||||
|
|
||||||
# 处理片段
|
|
||||||
rel_path = os.path.relpath(fp, project_folder)
|
|
||||||
for i, frag in enumerate(paper_fragments):
|
|
||||||
if frag.strip():
|
|
||||||
fragments.append(FileFragment(
|
|
||||||
file_path=fp,
|
|
||||||
content=frag,
|
|
||||||
rel_path=rel_path,
|
|
||||||
fragment_index=i,
|
|
||||||
total_fragments=len(paper_fragments)
|
|
||||||
))
|
|
||||||
|
|
||||||
mutable_status[2] = "处理完成"
|
|
||||||
return fragments
|
|
||||||
|
|
||||||
except TimeoutError as e:
|
|
||||||
self.failed_files.append((fp, "处理超时"))
|
|
||||||
mutable_status[2] = "处理超时"
|
|
||||||
return []
|
|
||||||
except Exception as e:
|
|
||||||
self.failed_files.append((fp, f"处理失败：{str(e)}"))
|
|
||||||
mutable_status[2] = "处理异常"
|
|
||||||
return []
|
|
||||||
finally:
|
|
||||||
timer.cancel()
|
|
||||||
|
|
||||||
def prepare_fragments(self, project_folder: str, file_paths: List[str]) -> Generator:
|
|
||||||
import concurrent.futures
|
|
||||||
|
|
||||||
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from typing import Generator, List
|
|
||||||
"""并行准备所有文件的处理片段"""
|
|
||||||
all_fragments = []
|
|
||||||
total_files = len(file_paths)
|
|
||||||
|
|
||||||
# 配置参数
|
|
||||||
self.refresh_interval = 0.2 # UI刷新间隔
|
|
||||||
self.watch_dog_patience = 5 # 看门狗超时时间
|
|
||||||
self.max_file_size = 10 * 1024 * 1024 # 10MB限制
|
|
||||||
self.max_workers = min(32, len(file_paths)) # 最多32个线程
|
|
||||||
|
|
||||||
# 创建有超时控制的线程池
|
|
||||||
executor = ThreadPoolExecutor(max_workers=self.max_workers)
|
|
||||||
|
|
||||||
# 用于跨线程状态传递的可变列表 - 增加文件名信息
|
|
||||||
mutable_status_array = [["等待中", time.time(), "pending", file_path] for file_path in file_paths]
|
|
||||||
|
|
||||||
# 创建文件处理任务
|
|
||||||
file_infos = [(fp, project_folder) for fp in file_paths]
|
|
||||||
|
|
||||||
# 提交所有任务，使用带超时控制的处理函数
|
|
||||||
futures = [
|
|
||||||
executor.submit(
|
|
||||||
self._process_single_file_with_timeout,
|
|
||||||
file_info,
|
|
||||||
mutable_status_array[i]
|
|
||||||
) for i, file_info in enumerate(file_infos)
|
|
||||||
]
|
|
||||||
|
|
||||||
# 更新UI的计数器
|
|
||||||
cnt = 0
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 监控任务执行
|
|
||||||
while True:
|
|
||||||
time.sleep(self.refresh_interval)
|
|
||||||
cnt += 1
|
|
||||||
|
|
||||||
# 检查任务完成状态
|
|
||||||
worker_done = [f.done() for f in futures]
|
|
||||||
|
|
||||||
# 更新状态显示
|
|
||||||
status_str = ""
|
|
||||||
for i, (status, timestamp, desc, file_path) in enumerate(mutable_status_array):
|
|
||||||
# 获取文件名（去掉路径）
|
|
||||||
file_name = os.path.basename(file_path)
|
|
||||||
if worker_done[i]:
|
|
||||||
status_str += f"文件 {file_name}: {desc}\n"
|
|
||||||
else:
|
|
||||||
status_str += f"文件 {file_name}: {status} {desc}\n"
|
|
||||||
|
|
||||||
# 更新UI
|
|
||||||
self.chatbot[-1] = [
|
|
||||||
"处理进度",
|
|
||||||
f"正在处理文件...\n\n{status_str}" + "." * (cnt % 10 + 1)
|
|
||||||
]
|
|
||||||
yield from update_ui(chatbot=self.chatbot, history=self.history)
|
|
||||||
|
|
||||||
# 检查是否所有任务完成
|
|
||||||
if all(worker_done):
|
|
||||||
break
|
|
||||||
|
|
||||||
finally:
|
|
||||||
# 确保线程池正确关闭
|
|
||||||
executor.shutdown(wait=False)
|
|
||||||
|
|
||||||
# 收集结果
|
|
||||||
processed_files = 0
|
|
||||||
for future in futures:
|
|
||||||
try:
|
|
||||||
fragments = future.result(timeout=0.1) # 给予一个短暂的超时时间来获取结果
|
|
||||||
all_fragments.extend(fragments)
|
|
||||||
processed_files += 1
|
|
||||||
except concurrent.futures.TimeoutError:
|
|
||||||
# 处理获取结果超时
|
|
||||||
file_index = futures.index(future)
|
|
||||||
self.failed_files.append((file_paths[file_index], "结果获取超时"))
|
|
||||||
continue
|
|
||||||
except Exception as e:
|
|
||||||
# 处理其他异常
|
|
||||||
file_index = futures.index(future)
|
|
||||||
self.failed_files.append((file_paths[file_index], f"未知错误：{str(e)}"))
|
|
||||||
continue
|
|
||||||
|
|
||||||
# 最终进度更新
|
|
||||||
self.chatbot.append([
|
|
||||||
"文件处理完成",
|
|
||||||
f"成功处理 {len(all_fragments)} 个片段，失败 {len(self.failed_files)} 个文件"
|
|
||||||
])
|
|
||||||
yield from update_ui(chatbot=self.chatbot, history=self.history)
|
|
||||||
|
|
||||||
return all_fragments
|
|
||||||
|
|
||||||
def _process_fragments_batch(self, fragments: List[FileFragment]) -> Generator:
|
|
||||||
"""批量处理文件片段"""
|
|
||||||
from collections import defaultdict
|
|
||||||
batch_size = 64 # 每批处理的片段数
|
|
||||||
max_retries = 3 # 最大重试次数
|
|
||||||
retry_delay = 5 # 重试延迟（秒）
|
|
||||||
results = defaultdict(list)
|
|
||||||
|
|
||||||
# 按批次处理
|
|
||||||
for i in range(0, len(fragments), batch_size):
|
|
||||||
batch = fragments[i:i + batch_size]
|
|
||||||
|
|
||||||
inputs_array, inputs_show_user_array, history_array = self._create_batch_inputs(batch)
|
|
||||||
sys_prompt_array = ["请总结以下内容："] * len(batch)
|
|
||||||
|
|
||||||
# 添加重试机制
|
|
||||||
for retry in range(max_retries):
|
|
||||||
try:
|
|
||||||
response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
|
||||||
inputs_array=inputs_array,
|
|
||||||
inputs_show_user_array=inputs_show_user_array,
|
|
||||||
llm_kwargs=self.llm_kwargs,
|
|
||||||
chatbot=self.chatbot,
|
|
||||||
history_array=history_array,
|
|
||||||
sys_prompt_array=sys_prompt_array,
|
|
||||||
)
|
|
||||||
|
|
||||||
# 处理响应
|
|
||||||
for j, frag in enumerate(batch):
|
|
||||||
summary = response_collection[j * 2 + 1]
|
|
||||||
if summary and summary.strip():
|
|
||||||
results[frag.rel_path].append({
|
|
||||||
'index': frag.fragment_index,
|
|
||||||
'summary': summary,
|
|
||||||
'total': frag.total_fragments
|
|
||||||
})
|
|
||||||
break # 成功处理，跳出重试循环
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
if retry == max_retries - 1: # 最后一次重试失败
|
|
||||||
for frag in batch:
|
|
||||||
self.failed_files.append((frag.file_path, f"处理失败：{str(e)}"))
|
|
||||||
else:
|
|
||||||
yield from update_ui(self.chatbot.append([f"批次处理失败，{retry_delay}秒后重试...", str(e)]))
|
|
||||||
time.sleep(retry_delay)
|
|
||||||
|
|
||||||
return results
|
|
||||||
|
|
||||||
def _generate_final_summary_request(self) -> Tuple[List, List, List]:
|
|
||||||
"""准备最终总结请求"""
|
|
||||||
if not self.file_summaries_map:
|
|
||||||
return (["无可用的文件总结"], ["生成最终总结"], [[]])
|
|
||||||
|
|
||||||
summaries = list(self.file_summaries_map.values())
|
|
||||||
if all(not summary for summary in summaries):
|
|
||||||
return (["所有文件处理均失败"], ["生成最终总结"], [[]])
|
|
||||||
|
|
||||||
if self.plugin_kwargs.get("advanced_arg"):
|
|
||||||
i_say = "根据以上所有文件的处理结果，按要求进行综合处理：" + self.plugin_kwargs['advanced_arg']
|
|
||||||
else:
|
|
||||||
i_say = "请根据以上所有文件的处理结果，生成最终的总结，不超过1000字。"
|
|
||||||
|
|
||||||
return ([i_say], [i_say], [summaries])
|
|
||||||
|
|
||||||
def process_files(self, project_folder: str, file_paths: List[str]) -> Generator:
|
|
||||||
"""处理所有文件"""
|
|
||||||
total_files = len(file_paths)
|
|
||||||
self.chatbot.append([f"开始处理", f"总计 {total_files} 个文件"])
|
|
||||||
yield from update_ui(chatbot=self.chatbot, history=self.history)
|
|
||||||
|
|
||||||
# 1. 准备所有文件片段
|
|
||||||
# 在 process_files 函数中：
|
|
||||||
fragments = yield from self.prepare_fragments(project_folder, file_paths)
|
|
||||||
if not fragments:
|
|
||||||
self.chatbot.append(["处理失败", "没有可处理的文件内容"])
|
|
||||||
return "没有可处理的文件内容"
|
|
||||||
|
|
||||||
# 2. 批量处理所有文件片段
|
|
||||||
self.chatbot.append([f"文件分析", f"共计 {len(fragments)} 个处理单元"])
|
|
||||||
yield from update_ui(chatbot=self.chatbot, history=self.history)
|
|
||||||
|
|
||||||
try:
|
|
||||||
file_summaries = yield from self._process_fragments_batch(fragments)
|
|
||||||
except Exception as e:
|
|
||||||
self.chatbot.append(["处理错误", f"批处理过程失败：{str(e)}"])
|
|
||||||
return "处理过程发生错误"
|
|
||||||
|
|
||||||
# 3. 为每个文件生成整体总结
|
|
||||||
self.chatbot.append(["生成总结", "正在汇总文件内容..."])
|
|
||||||
yield from update_ui(chatbot=self.chatbot, history=self.history)
|
|
||||||
|
|
||||||
# 处理每个文件的总结
|
|
||||||
for rel_path, summaries in file_summaries.items():
|
|
||||||
if len(summaries) > 1: # 多片段文件需要生成整体总结
|
|
||||||
sorted_summaries = sorted(summaries, key=lambda x: x['index'])
|
|
||||||
if self.plugin_kwargs.get("advanced_arg"):
|
|
||||||
|
|
||||||
i_say = f'请按照用户要求对文件内容进行处理，用户要求为：{self.plugin_kwargs["advanced_arg"]}：'
|
|
||||||
else:
|
|
||||||
i_say = f"请总结文件 {os.path.basename(rel_path)} 的主要内容，不超过500字。"
|
|
||||||
|
|
||||||
try:
|
|
||||||
summary_texts = [s['summary'] for s in sorted_summaries]
|
|
||||||
response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
|
||||||
inputs_array=[i_say],
|
|
||||||
inputs_show_user_array=[f"生成 {rel_path} 的处理结果"],
|
|
||||||
llm_kwargs=self.llm_kwargs,
|
|
||||||
chatbot=self.chatbot,
|
|
||||||
history_array=[summary_texts],
|
|
||||||
sys_prompt_array=["你是一个优秀的助手，"],
|
|
||||||
)
|
|
||||||
self.file_summaries_map[rel_path] = response_collection[1]
|
|
||||||
except Exception as e:
|
|
||||||
self.chatbot.append(["警告", f"文件 {rel_path} 总结生成失败：{str(e)}"])
|
|
||||||
self.file_summaries_map[rel_path] = "总结生成失败"
|
|
||||||
else: # 单片段文件直接使用其唯一的总结
|
|
||||||
self.file_summaries_map[rel_path] = summaries[0]['summary']
|
|
||||||
|
|
||||||
# 4. 生成最终总结
|
|
||||||
if total_files ==1:
|
|
||||||
return "文件数为1，此时不调用总结模块"
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
# 收集所有文件的总结用于生成最终总结
|
|
||||||
file_summaries_for_final = []
|
|
||||||
for rel_path, summary in self.file_summaries_map.items():
|
|
||||||
file_summaries_for_final.append(f"文件 {rel_path} 的总结：\n{summary}")
|
|
||||||
|
|
||||||
if self.plugin_kwargs.get("advanced_arg"):
|
|
||||||
final_summary_prompt = ("根据以下所有文件的总结内容，按要求进行综合处理：" +
|
|
||||||
self.plugin_kwargs['advanced_arg'])
|
|
||||||
else:
|
|
||||||
final_summary_prompt = "请根据以下所有文件的总结内容，生成最终的总结报告。"
|
|
||||||
|
|
||||||
response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
|
||||||
inputs_array=[final_summary_prompt],
|
|
||||||
inputs_show_user_array=["生成最终总结报告"],
|
|
||||||
llm_kwargs=self.llm_kwargs,
|
|
||||||
chatbot=self.chatbot,
|
|
||||||
history_array=[file_summaries_for_final],
|
|
||||||
sys_prompt_array=["总结所有文件内容。"],
|
|
||||||
max_workers=1
|
|
||||||
)
|
|
||||||
|
|
||||||
return response_collection[1] if len(response_collection) > 1 else "生成总结失败"
|
|
||||||
except Exception as e:
|
|
||||||
self.chatbot.append(["错误", f"最终总结生成失败：{str(e)}"])
|
|
||||||
return "生成总结失败"
|
|
||||||
|
|
||||||
def save_results(self, final_summary: str):
|
|
||||||
"""保存结果到文件"""
|
|
||||||
from toolbox import promote_file_to_downloadzone, write_history_to_file
|
|
||||||
from crazy_functions.doc_fns.batch_file_query_doc import MarkdownFormatter, HtmlFormatter, WordFormatter
|
|
||||||
import os
|
|
||||||
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
|
||||||
|
|
||||||
# 创建各种格式化器
|
|
||||||
md_formatter = MarkdownFormatter(final_summary, self.file_summaries_map, self.failed_files)
|
|
||||||
html_formatter = HtmlFormatter(final_summary, self.file_summaries_map, self.failed_files)
|
|
||||||
word_formatter = WordFormatter(final_summary, self.file_summaries_map, self.failed_files)
|
|
||||||
|
|
||||||
result_files = []
|
|
||||||
|
|
||||||
# 保存 Markdown
|
|
||||||
md_content = md_formatter.create_document()
|
|
||||||
result_file_md = write_history_to_file(
|
|
||||||
history=[md_content], # 直接传入内容列表
|
|
||||||
file_basename=f"文档总结_{timestamp}.md"
|
|
||||||
)
|
|
||||||
result_files.append(result_file_md)
|
|
||||||
|
|
||||||
# 保存 HTML
|
|
||||||
html_content = html_formatter.create_document()
|
|
||||||
result_file_html = write_history_to_file(
|
|
||||||
history=[html_content],
|
|
||||||
file_basename=f"文档总结_{timestamp}.html"
|
|
||||||
)
|
|
||||||
result_files.append(result_file_html)
|
|
||||||
|
|
||||||
# 保存 Word
|
|
||||||
doc = word_formatter.create_document()
|
|
||||||
# 由于 Word 文档需要用 doc.save()，我们使用与 md 文件相同的目录
|
|
||||||
result_file_docx = os.path.join(
|
|
||||||
os.path.dirname(result_file_md),
|
|
||||||
f"文档总结_{timestamp}.docx"
|
|
||||||
)
|
|
||||||
doc.save(result_file_docx)
|
|
||||||
result_files.append(result_file_docx)
|
|
||||||
|
|
||||||
# 添加到下载区
|
|
||||||
for file in result_files:
|
|
||||||
promote_file_to_downloadzone(file, chatbot=self.chatbot)
|
|
||||||
|
|
||||||
self.chatbot.append(["处理完成", f"结果已保存至: {', '.join(result_files)}"])
|
|
||||||
@CatchException
|
|
||||||
def 批量文件询问(txt: str, llm_kwargs: Dict, plugin_kwargs: Dict, chatbot: List,
|
|
||||||
history: List, system_prompt: str, user_request: str):
|
|
||||||
"""主函数 - 优化版本"""
|
|
||||||
# 初始化
|
|
||||||
import glob
|
|
||||||
import re
|
|
||||||
from crazy_functions.rag_fns.rag_file_support import supports_format
|
|
||||||
from toolbox import report_exception
|
|
||||||
|
|
||||||
summarizer = BatchDocumentSummarizer(llm_kwargs, plugin_kwargs, chatbot, history, system_prompt)
|
|
||||||
chatbot.append(["函数插件功能", f"作者：lbykkkk，批量总结文件。支持格式: {', '.join(supports_format)}等其他文本格式文件，如果长时间卡在文件处理过程，请查看处理进度，然后删除所有处于“pending”状态的文件，然后重新上传处理。"])
|
|
||||||
yield from update_ui(chatbot=chatbot, history=history)
|
|
||||||
|
|
||||||
# 验证输入路径
|
|
||||||
if not os.path.exists(txt):
|
|
||||||
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到项目或无权访问: {txt}")
|
|
||||||
yield from update_ui(chatbot=chatbot, history=history)
|
|
||||||
return
|
|
||||||
|
|
||||||
# 获取文件列表
|
|
||||||
project_folder = txt
|
|
||||||
extract_folder = next((d for d in glob.glob(f'{project_folder}/*')
|
|
||||||
if os.path.isdir(d) and d.endswith('.extract')), project_folder)
|
|
||||||
|
|
||||||
exclude_patterns = r'/[^/]+\.(zip|rar|7z|tar|gz)$'
|
|
||||||
file_manifest = [f for f in glob.glob(f'{extract_folder}/**', recursive=True)
|
|
||||||
if os.path.isfile(f) and not re.search(exclude_patterns, f)]
|
|
||||||
|
|
||||||
if not file_manifest:
|
|
||||||
report_exception(chatbot, history, a=f"解析项目: {txt}", b="未找到支持的文件类型")
|
|
||||||
yield from update_ui(chatbot=chatbot, history=history)
|
|
||||||
return
|
|
||||||
|
|
||||||
# 处理所有文件并生成总结
|
|
||||||
final_summary = yield from summarizer.process_files(project_folder, file_manifest)
|
|
||||||
yield from update_ui(chatbot=chatbot, history=history)
|
|
||||||
|
|
||||||
# 保存结果
|
|
||||||
summarizer.save_results(final_summary)
|
|
||||||
yield from update_ui(chatbot=chatbot, history=history)
|
|
||||||
@@ -180,7 +180,6 @@ version: '3'
|
|||||||
services:
|
services:
|
||||||
gpt_academic_with_latex:
|
gpt_academic_with_latex:
|
||||||
image: ghcr.io/binary-husky/gpt_academic_with_latex:master # (Auto Built by Dockerfile: docs/GithubAction+NoLocal+Latex)
|
image: ghcr.io/binary-husky/gpt_academic_with_latex:master # (Auto Built by Dockerfile: docs/GithubAction+NoLocal+Latex)
|
||||||
# 对于ARM64设备，请将以上镜像名称替换为 ghcr.io/binary-husky/gpt_academic_with_latex_arm:master
|
|
||||||
environment:
|
environment:
|
||||||
# 请查阅 `config.py` 以查看所有的配置信息
|
# 请查阅 `config.py` 以查看所有的配置信息
|
||||||
API_KEY: ' sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx '
|
API_KEY: ' sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx '
|
||||||
|
|||||||
@@ -1,34 +1,35 @@
|
|||||||
# 此Dockerfile适用于"无本地模型"的环境构建，如果需要使用chatglm等本地模型，请参考 docs/Dockerfile+ChatGLM
|
# 此Dockerfile适用于“无本地模型”的环境构建，如果需要使用chatglm等本地模型，请参考 docs/Dockerfile+ChatGLM
|
||||||
# - 1 修改 `config.py`
|
# - 1 修改 `config.py`
|
||||||
# - 2 构建 docker build -t gpt-academic-nolocal-latex -f docs/GithubAction+NoLocal+Latex .
|
# - 2 构建 docker build -t gpt-academic-nolocal-latex -f docs/GithubAction+NoLocal+Latex .
|
||||||
# - 3 运行 docker run -v /home/fuqingxu/arxiv_cache:/root/arxiv_cache --rm -it --net=host gpt-academic-nolocal-latex
|
# - 3 运行 docker run -v /home/fuqingxu/arxiv_cache:/root/arxiv_cache --rm -it --net=host gpt-academic-nolocal-latex
|
||||||
|
|
||||||
FROM menghuan1918/ubuntu_uv_ctex:latest
|
FROM fuqingxu/python311_texlive_ctex:latest
|
||||||
ENV DEBIAN_FRONTEND=noninteractive
|
ENV PATH "$PATH:/usr/local/texlive/2022/bin/x86_64-linux"
|
||||||
SHELL ["/bin/bash", "-c"]
|
ENV PATH "$PATH:/usr/local/texlive/2023/bin/x86_64-linux"
|
||||||
|
ENV PATH "$PATH:/usr/local/texlive/2024/bin/x86_64-linux"
|
||||||
|
ENV PATH "$PATH:/usr/local/texlive/2025/bin/x86_64-linux"
|
||||||
|
ENV PATH "$PATH:/usr/local/texlive/2026/bin/x86_64-linux"
|
||||||
|
|
||||||
|
# 指定路径
|
||||||
WORKDIR /gpt
|
WORKDIR /gpt
|
||||||
|
|
||||||
# 先复制依赖文件
|
RUN pip3 install openai numpy arxiv rich
|
||||||
COPY requirements.txt .
|
RUN pip3 install colorama Markdown pygments pymupdf
|
||||||
|
RUN pip3 install python-docx pdfminer
|
||||||
|
RUN pip3 install nougat-ocr
|
||||||
|
|
||||||
|
# 装载项目文件
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
|
||||||
# 安装依赖
|
# 安装依赖
|
||||||
RUN pip install --break-system-packages openai numpy arxiv rich colorama Markdown pygments pymupdf python-docx pdfminer \
|
RUN pip3 install -r requirements.txt
|
||||||
&& pip install --break-system-packages -r requirements.txt \
|
|
||||||
&& if [ "$(uname -m)" = "x86_64" ]; then \
|
|
||||||
pip install --break-system-packages nougat-ocr; \
|
|
||||||
fi \
|
|
||||||
&& pip cache purge \
|
|
||||||
&& rm -rf /root/.cache/pip/*
|
|
||||||
|
|
||||||
# 创建非root用户
|
# edge-tts需要的依赖
|
||||||
RUN useradd -m gptuser && chown -R gptuser /gpt
|
RUN apt update && apt install ffmpeg -y
|
||||||
USER gptuser
|
|
||||||
|
|
||||||
# 最后才复制代码文件,这样代码更新时只需重建最后几层，可以大幅减少docker pull所需的大小
|
|
||||||
COPY --chown=gptuser:gptuser . .
|
|
||||||
|
|
||||||
# 可选步骤，用于预热模块
|
# 可选步骤，用于预热模块
|
||||||
RUN python3 -c 'from check_proxy import warm_up_modules; warm_up_modules()'
|
RUN python3 -c 'from check_proxy import warm_up_modules; warm_up_modules()'
|
||||||
|
|
||||||
# 启动
|
# 启动
|
||||||
CMD ["python3", "-u", "main.py"]
|
CMD ["python3", "-u", "main.py"]
|
||||||
|
|||||||
25
docs/GithubAction+NoLocal+Latex+Arm
Normal file
25
docs/GithubAction+NoLocal+Latex+Arm
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
# 此Dockerfile适用于“无本地模型”的环境构建，如果需要使用chatglm等本地模型，请参考 docs/Dockerfile+ChatGLM
|
||||||
|
# - 1 修改 `config.py`
|
||||||
|
# - 2 构建 docker build -t gpt-academic-nolocal-latex -f docs/GithubAction+NoLocal+Latex .
|
||||||
|
# - 3 运行 docker run -v /home/fuqingxu/arxiv_cache:/root/arxiv_cache --rm -it --net=host gpt-academic-nolocal-latex
|
||||||
|
|
||||||
|
FROM menghuan1918/ubuntu_uv_ctex:latest
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
SHELL ["/bin/bash", "-c"]
|
||||||
|
WORKDIR /gpt
|
||||||
|
COPY . .
|
||||||
|
RUN /root/.cargo/bin/uv venv --seed \
|
||||||
|
&& source .venv/bin/activate \
|
||||||
|
&& /root/.cargo/bin/uv pip install openai numpy arxiv rich colorama Markdown pygments pymupdf python-docx pdfminer \
|
||||||
|
&& /root/.cargo/bin/uv pip install -r requirements.txt \
|
||||||
|
&& /root/.cargo/bin/uv clean
|
||||||
|
|
||||||
|
# 对齐python3
|
||||||
|
RUN rm -f /usr/bin/python3 && ln -s /gpt/.venv/bin/python /usr/bin/python3
|
||||||
|
RUN rm -f /usr/bin/python && ln -s /gpt/.venv/bin/python /usr/bin/python
|
||||||
|
|
||||||
|
# 可选步骤，用于预热模块
|
||||||
|
RUN python3 -c 'from check_proxy import warm_up_modules; warm_up_modules()'
|
||||||
|
|
||||||
|
# 启动
|
||||||
|
CMD ["python3", "-u", "main.py"]
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
import init_test
|
|
||||||
|
|
||||||
from crazy_functions.pdf_fns.parse_pdf_via_doc2x import 解析PDF_DOC2X_转Latex
|
|
||||||
|
|
||||||
# 解析PDF_DOC2X_转Latex("gpt_log/arxiv_cache_old/2410.10819/workfolder/merge.pdf")
|
|
||||||
# 解析PDF_DOC2X_转Latex("gpt_log/arxiv_cache_ooo/2410.07095/workfolder/merge.pdf")
|
|
||||||
解析PDF_DOC2X_转Latex("2410.11190v2.pdf")
|
|
||||||
Reference in New Issue
Block a user