diff --git a/README.md b/README.md index 62bb9cd1..87362913 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ Tip:不指定文件直接点击 `载入对话历史存档` 可以查看历史h 3. 虚空终端(从自然语言输入中,理解用户意图+自动调用其他插件) -- 步骤一:输入 “ 请调用插件翻译PDF论文,地址为https://storage.googleapis.com/deepmind-media/alphago/AlphaGoNaturePaper.pdf ” +- 步骤一:输入 “ 请调用插件翻译PDF论文,地址为https://openreview.net/pdf?id=rJl0r3R9KX ” - 步骤二:点击“虚空终端”
diff --git a/app.py b/app.py index 2da18793..3b96a097 100644 --- a/app.py +++ b/app.py @@ -9,7 +9,7 @@ def main(): # 建议您复制一个config_private.py放自己的秘密, 如API和代理网址, 避免不小心传github被别人看到 proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION = get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION') CHATBOT_HEIGHT, LAYOUT, AVAIL_LLM_MODELS, AUTO_CLEAR_TXT = get_conf('CHATBOT_HEIGHT', 'LAYOUT', 'AVAIL_LLM_MODELS', 'AUTO_CLEAR_TXT') - ENABLE_AUDIO, AUTO_CLEAR_TXT = get_conf('ENABLE_AUDIO', 'AUTO_CLEAR_TXT') + ENABLE_AUDIO, AUTO_CLEAR_TXT, PATH_LOGGING = get_conf('ENABLE_AUDIO', 'AUTO_CLEAR_TXT', 'PATH_LOGGING') # 如果WEB_PORT是-1, 则随机选取WEB端口 PORT = find_free_port() if WEB_PORT <= 0 else WEB_PORT @@ -22,12 +22,12 @@ def main(): # 问询记录, python 版本建议3.9+(越新越好) import logging, uuid - os.makedirs("gpt_log", exist_ok=True) - try:logging.basicConfig(filename="gpt_log/chat_secrets.log", level=logging.INFO, encoding="utf-8", format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") - except:logging.basicConfig(filename="gpt_log/chat_secrets.log", level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + os.makedirs(PATH_LOGGING, exist_ok=True) + try:logging.basicConfig(filename=f"{PATH_LOGGING}/chat_secrets.log", level=logging.INFO, encoding="utf-8", format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + except:logging.basicConfig(filename=f"{PATH_LOGGING}/chat_secrets.log", level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") # Disable logging output from the 'httpx' logger logging.getLogger("httpx").setLevel(logging.WARNING) - print("所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log, 请注意自我隐私保护哦!") + print(f"所有问询记录将自动保存在本地目录./{PATH_LOGGING}/chat_secrets.log, 请注意自我隐私保护哦!") # 一些普通功能模块 from core_functional import get_core_functions @@ -125,6 +125,15 @@ def main(): max_length_sl = gr.Slider(minimum=256, maximum=8192, value=4096, step=1, interactive=True, label="Local LLM MaxLength",) checkboxes = gr.CheckboxGroup(["基础功能区", "函数插件区", "底部输入区", "输入清除键", "插件参数区"], value=["基础功能区", "函数插件区"], label="显示/隐藏功能区") md_dropdown = gr.Dropdown(AVAIL_LLM_MODELS, value=LLM_MODEL, label="更换LLM模型/请求源").style(container=False) + dark_mode_btn = gr.Button("Toggle Dark Mode ☀", variant="secondary").style(size="sm") + dark_mode_btn.click(None, None, None, _js="""() => { + if (document.querySelectorAll('.dark').length) { + document.querySelectorAll('.dark').forEach(el => el.classList.remove('dark')); + } else { + document.querySelector('body').classList.add('dark'); + } + }""", + ) gr.Markdown(description) with gr.Accordion("备选输入区", open=True, visible=False, elem_id="input-panel2") as area_input_secondary: with gr.Row(): @@ -152,7 +161,7 @@ def main(): # 整理反复出现的控件句柄组合 input_combo = [cookies, max_length_sl, md_dropdown, txt, txt2, top_p, temperature, chatbot, history, system_prompt, plugin_advanced_arg] output_combo = [cookies, chatbot, history, status] - predict_args = dict(fn=ArgsGeneralWrapper(predict), inputs=input_combo, outputs=output_combo) + predict_args = dict(fn=ArgsGeneralWrapper(predict), inputs=[*input_combo, gr.State(True)], outputs=output_combo) # 提交按钮、重置按钮 cancel_handles.append(txt.submit(**predict_args)) cancel_handles.append(txt2.submit(**predict_args)) @@ -177,7 +186,7 @@ def main(): # 函数插件-固定按钮区 for k in plugins: if not plugins[k].get("AsButton", True): continue - click_handle = plugins[k]["Button"].click(ArgsGeneralWrapper(plugins[k]["Function"]), [*input_combo, gr.State(PORT)], output_combo) + click_handle = plugins[k]["Button"].click(ArgsGeneralWrapper(plugins[k]["Function"]), [*input_combo], output_combo) click_handle.then(on_report_generated, [cookies, file_upload, chatbot], [cookies, file_upload, chatbot]) cancel_handles.append(click_handle) # 函数插件-下拉菜单与随变按钮的互动 @@ -197,7 +206,7 @@ def main(): def route(request: gr.Request, k, *args, **kwargs): if k in [r"打开插件列表", r"请先从插件列表中选择"]: return yield from ArgsGeneralWrapper(plugins[k]["Function"])(request, *args, **kwargs) - click_handle = switchy_bt.click(route,[switchy_bt, *input_combo, gr.State(PORT)], output_combo) + click_handle = switchy_bt.click(route,[switchy_bt, *input_combo], output_combo) click_handle.then(on_report_generated, [cookies, file_upload, chatbot], [cookies, file_upload, chatbot]) cancel_handles.append(click_handle) # 终止按钮的回调函数注册 diff --git a/config.py b/config.py index 68b79614..94564fa0 100644 --- a/config.py +++ b/config.py @@ -47,8 +47,9 @@ API_URL_REDIRECT = {} DEFAULT_WORKER_NUM = 3 -# 色彩主题,可选 ["Default", "Chuanhu-Small-and-Beautiful", "High-Contrast"] -THEME = "Chuanhu-Small-and-Beautiful" +# 色彩主题, 可选 ["Default", "Chuanhu-Small-and-Beautiful", "High-Contrast"] +# 更多主题, 请查阅Gradio主题商店: https://huggingface.co/spaces/gradio/theme-gallery 可选 ["Gstaff/Xkcd", "NoCrypt/Miku", ...] +THEME = "Default" # 对话窗的高度 (仅在LAYOUT="TOP-DOWN"时生效) @@ -185,7 +186,10 @@ GROBID_URLS = [ # 是否允许通过自然语言描述修改本页的配置,该功能具有一定的危险性,默认关闭 ALLOW_RESET_CONFIG = False - +# 临时的上传文件夹位置,请勿修改 +PATH_PRIVATE_UPLOAD = "private_upload" +# 日志文件夹的位置,请勿修改 +PATH_LOGGING = "gpt_log" """ 在线大模型配置关联关系示意图 diff --git a/crazy_functional.py b/crazy_functional.py index c6578554..4df53f58 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -13,7 +13,6 @@ def get_crazy_functions(): from crazy_functions.解析项目源代码 import 解析一个Java项目 from crazy_functions.解析项目源代码 import 解析一个前端项目 from crazy_functions.高级功能函数模板 import 高阶功能模板函数 - from crazy_functions.代码重写为全英文_多线程 import 全项目切换英文 from crazy_functions.Latex全文润色 import Latex英文润色 from crazy_functions.询问多个大语言模型 import 同时问询 from crazy_functions.解析项目源代码 import 解析一个Lua项目 @@ -400,12 +399,12 @@ def get_crazy_functions(): try: from crazy_functions.Langchain知识库 import 知识库问答 function_plugins.update({ - "构建知识库(请先上传文件素材)": { + "构建知识库(先上传文件素材,再运行此插件)": { "Group": "对话", "Color": "stop", "AsButton": False, "AdvancedArgs": True, - "ArgsReminder": "待注入的知识库名称id, 默认为default", + "ArgsReminder": "此处待注入的知识库名称id, 默认为default。文件进入知识库后可长期保存。可以通过再次调用本插件的方式,向知识库追加更多文档。", "Function": HotReload(知识库问答) } }) @@ -415,12 +414,12 @@ def get_crazy_functions(): try: from crazy_functions.Langchain知识库 import 读取知识库作答 function_plugins.update({ - "知识库问答": { + "知识库问答(构建知识库后,再运行此插件)": { "Group": "对话", "Color": "stop", "AsButton": False, "AdvancedArgs": True, - "ArgsReminder": "待提取的知识库名称id, 默认为default, 您需要首先调用构建知识库", + "ArgsReminder": "待提取的知识库名称id, 默认为default, 您需要构建知识库后再运行此插件。", "Function": HotReload(读取知识库作答) } }) diff --git a/crazy_functions/CodeInterpreter.py b/crazy_functions/CodeInterpreter.py index 3c970f35..283dd87a 100644 --- a/crazy_functions/CodeInterpreter.py +++ b/crazy_functions/CodeInterpreter.py @@ -1,6 +1,7 @@ from collections.abc import Callable, Iterable, Mapping from typing import Any -from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc, promote_file_to_downloadzone, clear_file_downloadzone +from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc +from toolbox import promote_file_to_downloadzone, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import input_clipping, try_install_deps from multiprocessing import Process, Pipe @@ -92,7 +93,7 @@ def gpt_interact_multi_step(txt, file_type, llm_kwargs, chatbot, history): def make_module(code): module_file = 'gpt_fn_' + gen_time_str().replace('-','_') - with open(f'gpt_log/{module_file}.py', 'w', encoding='utf8') as f: + with open(f'{get_log_folder()}/{module_file}.py', 'w', encoding='utf8') as f: f.write(code) def get_class_name(class_string): @@ -102,7 +103,7 @@ def make_module(code): return class_name class_name = get_class_name(code) - return f"gpt_log.{module_file}->{class_name}" + return f"{get_log_folder().replace('/', '.')}.{module_file}->{class_name}" def init_module_instance(module): import importlib @@ -171,7 +172,7 @@ def 虚空终端CodeInterpreter(txt, llm_kwargs, plugin_kwargs, chatbot, history file_type = file_path.split('.')[-1] # 粗心检查 - if 'private_upload' in txt: + if is_the_upload_folder(txt): chatbot.append([ "...", f"请在输入框内填写需求,然后再次点击该插件(文件路径 {file_path} 已经被记忆)" diff --git a/crazy_functions/Langchain知识库.py b/crazy_functions/Langchain知识库.py index 12735dfd..741a3d06 100644 --- a/crazy_functions/Langchain知识库.py +++ b/crazy_functions/Langchain知识库.py @@ -1,4 +1,4 @@ -from toolbox import CatchException, update_ui, ProxyNetworkActivate +from toolbox import CatchException, update_ui, ProxyNetworkActivate, update_ui_lastest_msg from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive, get_files_from_everything @@ -15,7 +15,12 @@ def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_pro web_port 当前软件运行的端口号 """ history = [] # 清空历史,以免输入溢出 - chatbot.append(("这是什么功能?", "[Local Message] 从一批文件(txt, md, tex)中读取数据构建知识库, 然后进行问答。")) + + # < --------------------读取参数--------------- > + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + kai_id = plugin_kwargs.get("advanced_arg", 'default') + + chatbot.append((f"向`{kai_id}`知识库中添加文件。", "[Local Message] 从一批文件(txt, md, tex)中读取数据构建知识库, 然后进行问答。")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # resolve deps @@ -24,17 +29,12 @@ def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_pro from langchain.embeddings.huggingface import HuggingFaceEmbeddings from .crazy_utils import knowledge_archive_interface except Exception as e: - chatbot.append( - ["依赖不足", - "导入依赖失败。正在尝试自动安装,请查看终端的输出或耐心等待..."] - ) + chatbot.append(["依赖不足", "导入依赖失败。正在尝试自动安装,请查看终端的输出或耐心等待..."]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 from .crazy_utils import try_install_deps - try_install_deps(['zh_langchain==0.2.1', 'pypinyin']) - - # < --------------------读取参数--------------- > - if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") - kai_id = plugin_kwargs.get("advanced_arg", 'default') + try_install_deps(['zh_langchain==0.2.1', 'pypinyin'], reload_m=['pypinyin', 'zh_langchain']) + yield from update_ui_lastest_msg("安装完成,您可以再次重试。", chatbot, history) + return # < --------------------读取文件--------------- > file_manifest = [] @@ -84,19 +84,18 @@ def 读取知识库作答(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst chatbot.append(["依赖不足", "导入依赖失败。正在尝试自动安装,请查看终端的输出或耐心等待..."]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 from .crazy_utils import try_install_deps - try_install_deps(['zh_langchain==0.2.1']) + try_install_deps(['zh_langchain==0.2.1', 'pypinyin'], reload_m=['pypinyin', 'zh_langchain']) + yield from update_ui_lastest_msg("安装完成,您可以再次重试。", chatbot, history) + return # < ------------------- --------------- > kai = knowledge_archive_interface() - if 'langchain_plugin_embedding' in chatbot._cookies: - resp, prompt = kai.answer_with_archive_by_id(txt, chatbot._cookies['langchain_plugin_embedding']) - else: - if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") - kai_id = plugin_kwargs.get("advanced_arg", 'default') - resp, prompt = kai.answer_with_archive_by_id(txt, kai_id) + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + kai_id = plugin_kwargs.get("advanced_arg", 'default') + resp, prompt = kai.answer_with_archive_by_id(txt, kai_id) - chatbot.append((txt, '[Local Message] ' + prompt)) + chatbot.append((txt, f'[知识库 {kai_id}] ' + prompt)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 由于请求gpt需要一段时间,我们先及时地做一次界面更新 gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive( inputs=prompt, inputs_show_user=txt, diff --git a/crazy_functions/Latex全文润色.py b/crazy_functions/Latex全文润色.py index 9e1d4b66..462f9657 100644 --- a/crazy_functions/Latex全文润色.py +++ b/crazy_functions/Latex全文润色.py @@ -1,5 +1,5 @@ -from toolbox import update_ui, trimmed_format_exc -from toolbox import CatchException, report_execption, write_results_to_file, zip_folder +from toolbox import update_ui, trimmed_format_exc, promote_file_to_downloadzone, get_log_folder +from toolbox import CatchException, report_execption, write_history_to_file, zip_folder class PaperFileGroup(): @@ -51,7 +51,7 @@ class PaperFileGroup(): import os, time folder = os.path.dirname(self.file_paths[0]) t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) - zip_folder(folder, './gpt_log/', f'{t}-polished.zip') + zip_folder(folder, get_log_folder(), f'{t}-polished.zip') def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en', mode='polish'): @@ -126,7 +126,9 @@ def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch # <-------- 整理结果,退出 ----------> create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" - res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) + res = write_history_to_file(gpt_response_collection, file_basename=create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) + history = gpt_response_collection chatbot.append((f"{fp}完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 @@ -137,7 +139,7 @@ def Latex英文润色(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p # 基本信息:功能、贡献者 chatbot.append([ "函数插件功能?", - "对整个Latex项目进行润色。函数插件贡献者: Binary-Husky"]) + "对整个Latex项目进行润色。函数插件贡献者: Binary-Husky。(注意,此插件不调用Latex,如果有Latex环境,请使用“Latex英文纠错+高亮”插件)"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 尝试导入依赖,如果缺少依赖,则给出安装建议 diff --git a/crazy_functions/Latex全文翻译.py b/crazy_functions/Latex全文翻译.py index 554c485a..b5aad71b 100644 --- a/crazy_functions/Latex全文翻译.py +++ b/crazy_functions/Latex全文翻译.py @@ -1,5 +1,5 @@ -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import update_ui, promote_file_to_downloadzone +from toolbox import CatchException, report_execption, write_history_to_file fast_debug = False class PaperFileGroup(): @@ -95,7 +95,8 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch # <-------- 整理结果,退出 ----------> create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" - res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) + res = write_history_to_file(gpt_response_collection, create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) history = gpt_response_collection chatbot.append((f"{fp}完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/Latex输出PDF结果.py b/crazy_functions/Latex输出PDF结果.py index 8686f7e9..56ca1c6c 100644 --- a/crazy_functions/Latex输出PDF结果.py +++ b/crazy_functions/Latex输出PDF结果.py @@ -1,4 +1,4 @@ -from toolbox import update_ui, trimmed_format_exc, get_conf, objdump, objload, promote_file_to_downloadzone +from toolbox import update_ui, trimmed_format_exc, get_conf, get_log_folder, promote_file_to_downloadzone from toolbox import CatchException, report_execption, update_ui_lastest_msg, zip_result, gen_time_str from functools import partial import glob, os, requests, time @@ -65,7 +65,7 @@ def move_project(project_folder, arxiv_id=None): if arxiv_id is not None: new_workfolder = pj(ARXIV_CACHE_DIR, arxiv_id, 'workfolder') else: - new_workfolder = f'gpt_log/{gen_time_str()}' + new_workfolder = f'{get_log_folder()}/{gen_time_str()}' try: shutil.rmtree(new_workfolder) except: diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 5a314b37..ee1ab907 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -1,5 +1,7 @@ -from toolbox import update_ui, get_conf, trimmed_format_exc +from toolbox import update_ui, get_conf, trimmed_format_exc, get_log_folder import threading +import os +import logging def input_clipping(inputs, history, max_token_limit): import numpy as np @@ -469,14 +471,16 @@ def read_and_clean_pdf_text(fp): '- ', '') for t in text_areas['blocks'] if 'lines' in t] ############################## <第 2 步,获取正文主字体> ################################## - fsize_statiscs = {} - for span in meta_span: - if span[1] not in fsize_statiscs: fsize_statiscs[span[1]] = 0 - fsize_statiscs[span[1]] += span[2] - main_fsize = max(fsize_statiscs, key=fsize_statiscs.get) - if REMOVE_FOOT_NOTE: - give_up_fize_threshold = main_fsize * REMOVE_FOOT_FFSIZE_PERCENT - + try: + fsize_statiscs = {} + for span in meta_span: + if span[1] not in fsize_statiscs: fsize_statiscs[span[1]] = 0 + fsize_statiscs[span[1]] += span[2] + main_fsize = max(fsize_statiscs, key=fsize_statiscs.get) + if REMOVE_FOOT_NOTE: + give_up_fize_threshold = main_fsize * REMOVE_FOOT_FFSIZE_PERCENT + except: + raise RuntimeError(f'抱歉, 我们暂时无法解析此PDF文档: {fp}。') ############################## <第 3 步,切分和重新整合> ################################## mega_sec = [] sec = [] @@ -703,49 +707,96 @@ class knowledge_archive_interface(): ) self.threadLock.release() return resp, prompt + +@Singleton +class nougat_interface(): + def __init__(self): + self.threadLock = threading.Lock() -def try_install_deps(deps): + def nougat_with_timeout(self, command, cwd, timeout=3600): + import subprocess + logging.info(f'正在执行命令 {command}') + process = subprocess.Popen(command, shell=True, cwd=cwd) + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + print("Process timed out!") + return False + return True + + + def NOUGAT_parse_pdf(self, fp, chatbot, history): + from toolbox import update_ui_lastest_msg + + yield from update_ui_lastest_msg("正在解析论文, 请稍候。进度:正在排队, 等待线程锁...", + chatbot=chatbot, history=history, delay=0) + self.threadLock.acquire() + import glob, threading, os + from toolbox import get_log_folder, gen_time_str + dst = os.path.join(get_log_folder(plugin_name='nougat'), gen_time_str()) + os.makedirs(dst) + + yield from update_ui_lastest_msg("正在解析论文, 请稍候。进度:正在加载NOUGAT... (提示:首次运行需要花费较长时间下载NOUGAT参数)", + chatbot=chatbot, history=history, delay=0) + self.nougat_with_timeout(f'nougat --out "{os.path.abspath(dst)}" "{os.path.abspath(fp)}"', os.getcwd(), timeout=3600) + res = glob.glob(os.path.join(dst,'*.mmd')) + if len(res) == 0: + self.threadLock.release() + raise RuntimeError("Nougat解析论文失败。") + self.threadLock.release() + return res[0] + + + + +def try_install_deps(deps, reload_m=[]): + import subprocess, sys, importlib for dep in deps: - import subprocess, sys subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', dep]) + import site + importlib.reload(site) + for m in reload_m: + importlib.reload(__import__(m)) -class construct_html(): - def __init__(self) -> None: - self.css = """ +HTML_CSS = """ .row { display: flex; flex-wrap: wrap; } - .column { flex: 1; padding: 10px; } - .table-header { font-weight: bold; border-bottom: 1px solid black; } - .table-row { border-bottom: 1px solid lightgray; } - .table-cell { padding: 5px; } - """ - self.html_string = f'翻译结果' +""" - - def add_row(self, a, b): - tmp = """ +TABLE_CSS = """
REPLACE_A
REPLACE_B
- """ +""" + +class construct_html(): + def __init__(self) -> None: + self.css = HTML_CSS + self.html_string = f'翻译结果' + + + def add_row(self, a, b): + tmp = TABLE_CSS from toolbox import markdown_convertion tmp = tmp.replace('REPLACE_A', markdown_convertion(a)) tmp = tmp.replace('REPLACE_B', markdown_convertion(b)) @@ -753,6 +804,6 @@ class construct_html(): def save_file(self, file_name): - with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: + with open(os.path.join(get_log_folder(), file_name), 'w', encoding='utf8') as f: f.write(self.html_string.encode('utf-8', 'ignore').decode()) - + return os.path.join(get_log_folder(), file_name) diff --git a/crazy_functions/latex_fns/latex_actions.py b/crazy_functions/latex_fns/latex_actions.py index c59bc31d..dcde0e99 100644 --- a/crazy_functions/latex_fns/latex_actions.py +++ b/crazy_functions/latex_fns/latex_actions.py @@ -1,4 +1,4 @@ -from toolbox import update_ui, update_ui_lastest_msg # 刷新Gradio前端界面 +from toolbox import update_ui, update_ui_lastest_msg, get_log_folder from toolbox import zip_folder, objdump, objload, promote_file_to_downloadzone from .latex_toolbox import PRESERVE, TRANSFORM from .latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace @@ -363,7 +363,7 @@ def 编译Latex(chatbot, history, main_file_original, main_file_modified, work_f if mode!='translate_zh': yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 使用latexdiff生成论文转化前后对比 ...', chatbot, history) # 刷新Gradio前端界面 print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') - ok = compile_latex_with_timeout(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + ok = compile_latex_with_timeout(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex', os.getcwd()) yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 正在编译对比PDF ...', chatbot, history) # 刷新Gradio前端界面 ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex', work_folder) @@ -439,9 +439,9 @@ def write_html(sp_file_contents, sp_file_result, chatbot, project_folder): trans = k ch.add_row(a=orig, b=trans) create_report_file_name = f"{gen_time_str()}.trans.html" - ch.save_file(create_report_file_name) - shutil.copyfile(pj('./gpt_log/', create_report_file_name), pj(project_folder, create_report_file_name)) - promote_file_to_downloadzone(file=f'./gpt_log/{create_report_file_name}', chatbot=chatbot) + res = ch.save_file(create_report_file_name) + shutil.copyfile(res, pj(project_folder, create_report_file_name)) + promote_file_to_downloadzone(file=res, chatbot=chatbot) except: from toolbox import trimmed_format_exc print('writing html result failed:', trimmed_format_exc()) diff --git a/crazy_functions/latex_fns/latex_toolbox.py b/crazy_functions/latex_fns/latex_toolbox.py index 5adc7ea8..c733c818 100644 --- a/crazy_functions/latex_fns/latex_toolbox.py +++ b/crazy_functions/latex_fns/latex_toolbox.py @@ -256,6 +256,7 @@ def find_main_tex_file(file_manifest, mode): canidates_score.append(0) with open(texf, 'r', encoding='utf8', errors='ignore') as f: file_content = f.read() + file_content = rm_comments(file_content) for uw in unexpected_words: if uw in file_content: canidates_score[-1] -= 1 @@ -290,7 +291,11 @@ def find_tex_file_ignore_case(fp): import glob for f in glob.glob(dir_name+'/*.tex'): base_name_s = os.path.basename(fp) - if base_name_s.lower() == base_name.lower(): return f + base_name_f = os.path.basename(f) + if base_name_s.lower() == base_name_f.lower(): return f + # 试着加上.tex后缀试试 + if not base_name_s.endswith('.tex'): base_name_s+='.tex' + if base_name_s.lower() == base_name_f.lower(): return f return None def merge_tex_files_(project_foler, main_file, mode): @@ -301,9 +306,9 @@ def merge_tex_files_(project_foler, main_file, mode): for s in reversed([q for q in re.finditer(r"\\input\{(.*?)\}", main_file, re.M)]): f = s.group(1) fp = os.path.join(project_foler, f) - fp = find_tex_file_ignore_case(fp) - if fp: - with open(fp, 'r', encoding='utf-8', errors='replace') as fx: c = fx.read() + fp_ = find_tex_file_ignore_case(fp) + if fp_: + with open(fp_, 'r', encoding='utf-8', errors='replace') as fx: c = fx.read() else: raise RuntimeError(f'找不到{fp},Tex源文件缺失!') c = merge_tex_files_(project_foler, c, mode) @@ -423,7 +428,7 @@ def compile_latex_with_timeout(command, cwd, timeout=60): def merge_pdfs(pdf1_path, pdf2_path, output_path): import PyPDF2 - Percent = 0.8 + Percent = 0.95 # Open the first PDF file with open(pdf1_path, 'rb') as pdf1_file: pdf1_reader = PyPDF2.PdfFileReader(pdf1_file) diff --git a/crazy_functions/下载arxiv论文翻译摘要.py b/crazy_functions/下载arxiv论文翻译摘要.py index 46275bf4..8b4a5037 100644 --- a/crazy_functions/下载arxiv论文翻译摘要.py +++ b/crazy_functions/下载arxiv论文翻译摘要.py @@ -1,5 +1,6 @@ -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file, get_conf +from toolbox import update_ui, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone +from toolbox import CatchException, report_execption, get_conf import re, requests, unicodedata, os from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive def download_arxiv_(url_pdf): @@ -28,7 +29,7 @@ def download_arxiv_(url_pdf): if k in other_info['comment']: title = k + ' ' + title - download_dir = './gpt_log/arxiv/' + download_dir = get_log_folder(plugin_name='arxiv') os.makedirs(download_dir, exist_ok=True) title_str = title.replace('?', '?')\ @@ -40,9 +41,6 @@ def download_arxiv_(url_pdf): requests_pdf_url = url_pdf file_path = download_dir+title_str - # if os.path.exists(file_path): - # print('返回缓存文件') - # return './gpt_log/arxiv/'+title_str print('下载中') proxies, = get_conf('proxies') @@ -61,7 +59,7 @@ def download_arxiv_(url_pdf): .replace('\n', '')\ .replace(' ', ' ')\ .replace(' ', ' ') - return './gpt_log/arxiv/'+title_str, other_info + return file_path, other_info def get_name(_url_): @@ -184,11 +182,10 @@ def 下载arxiv论文并翻译摘要(txt, llm_kwargs, plugin_kwargs, chatbot, hi chatbot[-1] = (i_say_show_user, gpt_say) history.append(i_say_show_user); history.append(gpt_say) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - # 写入文件 - import shutil - # 重置文件的创建时间 - shutil.copyfile(pdf_path, f'./gpt_log/{os.path.basename(pdf_path)}'); os.remove(pdf_path) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) + promote_file_to_downloadzone(pdf_path, chatbot=chatbot) + chatbot.append(("完成了吗?", res + "\n\nPDF文件也已经下载")) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/图片生成.py b/crazy_functions/图片生成.py index a1cc2441..51a1baff 100644 --- a/crazy_functions/图片生成.py +++ b/crazy_functions/图片生成.py @@ -1,4 +1,4 @@ -from toolbox import CatchException, update_ui, get_conf, select_api_key +from toolbox import CatchException, update_ui, get_conf, select_api_key, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive import datetime @@ -33,7 +33,7 @@ def gen_image(llm_kwargs, prompt, resolution="256x256"): raise RuntimeError(response.content.decode()) # 文件保存到本地 r = requests.get(image_url, proxies=proxies) - file_path = 'gpt_log/image_gen/' + file_path = f'{get_log_folder()}/image_gen/' os.makedirs(file_path, exist_ok=True) file_name = 'Image' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.png' with open(file_path+file_name, 'wb+') as f: f.write(r.content) diff --git a/crazy_functions/对话历史存档.py b/crazy_functions/对话历史存档.py index a0d0ac8e..f89faeda 100644 --- a/crazy_functions/对话历史存档.py +++ b/crazy_functions/对话历史存档.py @@ -1,4 +1,4 @@ -from toolbox import CatchException, update_ui, promote_file_to_downloadzone +from toolbox import CatchException, update_ui, promote_file_to_downloadzone, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive import re @@ -10,8 +10,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None): import time if file_name is None: file_name = 'chatGPT对话历史' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html' - os.makedirs('./gpt_log/', exist_ok=True) - with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: + fp = os.path.join(get_log_folder(), file_name) + with open(fp, 'w', encoding='utf8') as f: from themes.theme import advanced_css f.write(f'对话历史') for i, contents in enumerate(chatbot): @@ -29,8 +29,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None): for h in history: f.write("\n>>>" + h) f.write('') - promote_file_to_downloadzone(f'./gpt_log/{file_name}', rename_file=file_name, chatbot=chatbot) - return '对话历史写入:' + os.path.abspath(f'./gpt_log/{file_name}') + promote_file_to_downloadzone(fp, rename_file=file_name, chatbot=chatbot) + return '对话历史写入:' + fp def gen_file_preview(file_name): try: @@ -106,7 +106,7 @@ def 载入对话历史存档(txt, llm_kwargs, plugin_kwargs, chatbot, history, s if not success: if txt == "": txt = '空空如也的输入栏' import glob - local_history = "
".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True)]) + local_history = "
".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)]) chatbot.append([f"正在查找对话历史文件(html格式): {txt}", f"找不到任何html文件: {txt}。但本地存储了以下历史文件,您可以将任意一个文件路径粘贴到输入区,然后重试:
{local_history}"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return @@ -132,8 +132,8 @@ def 删除所有本地对话历史记录(txt, llm_kwargs, plugin_kwargs, chatbot """ import glob, os - local_history = "
".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True)]) - for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True): + local_history = "
".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)]) + for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True): os.remove(f) chatbot.append([f"删除所有历史对话文件", f"已删除
{local_history}"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/总结word文档.py b/crazy_functions/总结word文档.py index 5af69607..4ea753cb 100644 --- a/crazy_functions/总结word文档.py +++ b/crazy_functions/总结word文档.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive fast_debug = False @@ -71,11 +72,13 @@ def 解析docx(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot history.extend([i_say,gpt_say]) this_paper_history.extend([i_say,gpt_say]) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("所有文件都总结完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/总结音视频.py b/crazy_functions/总结音视频.py index 62f05d39..7c113f47 100644 --- a/crazy_functions/总结音视频.py +++ b/crazy_functions/总结音视频.py @@ -1,5 +1,6 @@ -from toolbox import CatchException, report_execption, select_api_key, update_ui, write_results_to_file, get_conf +from toolbox import CatchException, report_execption, select_api_key, update_ui, get_conf from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +from toolbox import write_history_to_file, promote_file_to_downloadzone, get_log_folder def split_audio_file(filename, split_duration=1000): """ @@ -15,7 +16,7 @@ def split_audio_file(filename, split_duration=1000): """ from moviepy.editor import AudioFileClip import os - os.makedirs('gpt_log/mp3/cut/', exist_ok=True) # 创建存储切割音频的文件夹 + os.makedirs(f"{get_log_folder(plugin_name='audio')}/mp3/cut/", exist_ok=True) # 创建存储切割音频的文件夹 # 读取音频文件 audio = AudioFileClip(filename) @@ -31,8 +32,8 @@ def split_audio_file(filename, split_duration=1000): start_time = split_points[i] end_time = split_points[i + 1] split_audio = audio.subclip(start_time, end_time) - split_audio.write_audiofile(f"gpt_log/mp3/cut/{filename[0]}_{i}.mp3") - filelist.append(f"gpt_log/mp3/cut/{filename[0]}_{i}.mp3") + split_audio.write_audiofile(f"{get_log_folder(plugin_name='audio')}/mp3/cut/{filename[0]}_{i}.mp3") + filelist.append(f"{get_log_folder(plugin_name='audio')}/mp3/cut/{filename[0]}_{i}.mp3") audio.close() return filelist @@ -52,7 +53,7 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): 'Authorization': f"Bearer {api_key}" } - os.makedirs('gpt_log/mp3/', exist_ok=True) + os.makedirs(f"{get_log_folder(plugin_name='audio')}/mp3/", exist_ok=True) for index, fp in enumerate(file_manifest): audio_history = [] # 提取文件扩展名 @@ -60,8 +61,8 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): # 提取视频中的音频 if ext not in [".mp3", ".wav", ".m4a", ".mpga"]: audio_clip = AudioFileClip(fp) - audio_clip.write_audiofile(f'gpt_log/mp3/output{index}.mp3') - fp = f'gpt_log/mp3/output{index}.mp3' + audio_clip.write_audiofile(f"{get_log_folder(plugin_name='audio')}/mp3/output{index}.mp3") + fp = f"{get_log_folder(plugin_name='audio')}/mp3/output{index}.mp3" # 调用whisper模型音频转文字 voice = split_audio_file(fp) for j, i in enumerate(voice): @@ -113,18 +114,19 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): history=audio_history, sys_prompt="总结文章。" ) - history.extend([i_say, gpt_say]) audio_history.extend([i_say, gpt_say]) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append((f"第{index + 1}段音频完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 删除中间文件夹 import shutil - shutil.rmtree('gpt_log/mp3') - res = write_results_to_file(history) + shutil.rmtree(f"{get_log_folder(plugin_name='audio')}/mp3") + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("所有音频都总结完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) diff --git a/crazy_functions/批量Markdown翻译.py b/crazy_functions/批量Markdown翻译.py index 8af98640..9485b1ec 100644 --- a/crazy_functions/批量Markdown翻译.py +++ b/crazy_functions/批量Markdown翻译.py @@ -1,7 +1,7 @@ -import glob, time, os, re +import glob, time, os, re, logging from toolbox import update_ui, trimmed_format_exc, gen_time_str, disable_auto_promotion -from toolbox import CatchException, report_execption, write_history_to_file -from toolbox import promote_file_to_downloadzone, get_log_folder +from toolbox import CatchException, report_execption, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = False class PaperFileGroup(): @@ -34,7 +34,7 @@ class PaperFileGroup(): self.sp_file_contents.append(segment) self.sp_file_index.append(index) self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.md") - print('Segmentation: done') + logging.info('Segmentation: done') def merge_result(self): self.file_result = ["" for _ in range(len(self.file_paths))] @@ -101,7 +101,7 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch pfg.merge_result() pfg.write_result(language) except: - print(trimmed_format_exc()) + logging.error(trimmed_format_exc()) # <-------- 整理结果,退出 ----------> create_report_file_name = gen_time_str() + f"-chatgpt.md" @@ -121,7 +121,7 @@ def get_files_from_everything(txt, preference=''): proxies, = get_conf('proxies') # 网络的远程文件 if preference == 'Github': - print('正在从github下载资源 ...') + logging.info('正在从github下载资源 ...') if not txt.endswith('.md'): # Make a request to the GitHub API to retrieve the repository information url = txt.replace("https://github.com/", "https://api.github.com/repos/") + '/readme' diff --git a/crazy_functions/批量总结PDF文档.py b/crazy_functions/批量总结PDF文档.py index fc65f5c8..b87d4825 100644 --- a/crazy_functions/批量总结PDF文档.py +++ b/crazy_functions/批量总结PDF文档.py @@ -1,5 +1,6 @@ from toolbox import update_ui, promote_file_to_downloadzone, gen_time_str -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import read_and_clean_pdf_text from .crazy_utils import input_clipping @@ -99,8 +100,8 @@ do not have too much repetitive information, numerical values using the original _, final_results = input_clipping("", final_results, max_token_limit=3200) yield from update_ui(chatbot=chatbot, history=final_results) # 注意这里的历史记录被替代了 - res = write_results_to_file(file_write_buffer, file_name=gen_time_str()) - promote_file_to_downloadzone(res.split('\t')[-1], chatbot=chatbot) + res = write_history_to_file(file_write_buffer) + promote_file_to_downloadzone(res, chatbot=chatbot) yield from update_ui(chatbot=chatbot, history=final_results) # 刷新界面 diff --git a/crazy_functions/批量总结PDF文档pdfminer.py b/crazy_functions/批量总结PDF文档pdfminer.py index ffbb0559..213d8bb2 100644 --- a/crazy_functions/批量总结PDF文档pdfminer.py +++ b/crazy_functions/批量总结PDF文档pdfminer.py @@ -1,6 +1,7 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = False @@ -115,7 +116,8 @@ def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo chatbot[-1] = (i_say, gpt_say) history.append(i_say); history.append(gpt_say) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/批量翻译PDF文档_NOUGAT.py b/crazy_functions/批量翻译PDF文档_NOUGAT.py index ed151211..2dc15f79 100644 --- a/crazy_functions/批量翻译PDF文档_NOUGAT.py +++ b/crazy_functions/批量翻译PDF文档_NOUGAT.py @@ -86,31 +86,8 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst # 开始正式执行任务 yield from 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt) - -def nougat_with_timeout(command, cwd, timeout=3600): - import subprocess - process = subprocess.Popen(command, shell=True, cwd=cwd) - try: - stdout, stderr = process.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - process.kill() - stdout, stderr = process.communicate() - print("Process timed out!") - return False - return True -def NOUGAT_parse_pdf(fp): - import glob - from toolbox import get_log_folder, gen_time_str - dst = os.path.join(get_log_folder(plugin_name='nougat'), gen_time_str()) - os.makedirs(dst) - nougat_with_timeout(f'nougat --out "{os.path.abspath(dst)}" "{os.path.abspath(fp)}"', os.getcwd()) - res = glob.glob(os.path.join(dst,'*.mmd')) - if len(res) == 0: - raise RuntimeError("Nougat解析论文失败。") - return res[0] - def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): import copy @@ -119,9 +96,11 @@ def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwa generated_conclusion_files = [] generated_html_files = [] DST_LANG = "中文" + from crazy_functions.crazy_utils import nougat_interface, construct_html + nougat_handle = nougat_interface() for index, fp in enumerate(file_manifest): chatbot.append(["当前进度:", f"正在解析论文,请稍候。(第一次运行时,需要花费较长时间下载NOUGAT参数)"]); yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - fpp = NOUGAT_parse_pdf(fp) + fpp = yield from nougat_handle.NOUGAT_parse_pdf(fp, chatbot, history) with open(fpp, 'r', encoding='utf8') as f: article_content = f.readlines() @@ -222,50 +201,3 @@ def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwa yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - -class construct_html(): - def __init__(self) -> None: - self.css = """ -.row { - display: flex; - flex-wrap: wrap; -} - -.column { - flex: 1; - padding: 10px; -} - -.table-header { - font-weight: bold; - border-bottom: 1px solid black; -} - -.table-row { - border-bottom: 1px solid lightgray; -} - -.table-cell { - padding: 5px; -} - """ - self.html_string = f'翻译结果' - - - def add_row(self, a, b): - tmp = """ -
-
REPLACE_A
-
REPLACE_B
-
- """ - from toolbox import markdown_convertion - tmp = tmp.replace('REPLACE_A', markdown_convertion(a)) - tmp = tmp.replace('REPLACE_B', markdown_convertion(b)) - self.html_string += tmp - - - def save_file(self, file_name): - with open(os.path.join(get_log_folder(), file_name), 'w', encoding='utf8') as f: - f.write(self.html_string.encode('utf-8', 'ignore').decode()) - return os.path.join(get_log_folder(), file_name) diff --git a/crazy_functions/批量翻译PDF文档_多线程.py b/crazy_functions/批量翻译PDF文档_多线程.py index 6e9fe6a8..d620715b 100644 --- a/crazy_functions/批量翻译PDF文档_多线程.py +++ b/crazy_functions/批量翻译PDF文档_多线程.py @@ -1,6 +1,6 @@ -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption, get_log_folder from toolbox import update_ui, promote_file_to_downloadzone, update_ui_lastest_msg, disable_auto_promotion -from toolbox import write_history_to_file, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency from .crazy_utils import read_and_clean_pdf_text @@ -63,6 +63,7 @@ def 解析PDF_基于GROBID(file_manifest, project_folder, llm_kwargs, plugin_kwa generated_conclusion_files = [] generated_html_files = [] DST_LANG = "中文" + from crazy_functions.crazy_utils import construct_html for index, fp in enumerate(file_manifest): chatbot.append(["当前进度:", f"正在连接GROBID服务,请稍候: {grobid_url}\n如果等待时间过长,请修改config中的GROBID_URL,可修改成本地GROBID服务。"]); yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 article_dict = parse_pdf(fp, grobid_url) @@ -166,6 +167,7 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, TOKEN_LIMIT_PER_FRAGMENT = 1280 generated_conclusion_files = [] generated_html_files = [] + from crazy_functions.crazy_utils import construct_html for index, fp in enumerate(file_manifest): # 读取PDF文件 file_content, page_one = read_and_clean_pdf_text(fp) @@ -216,10 +218,11 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, final = ["一、论文概况\n\n---\n\n", paper_meta_info.replace('# ', '### ') + '\n\n---\n\n', "二、论文翻译", ""] final.extend(gpt_response_collection_md) create_report_file_name = f"{os.path.basename(fp)}.trans.md" - res = write_results_to_file(final, file_name=create_report_file_name) + res = write_history_to_file(final, create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) # 更新UI - generated_conclusion_files.append(f'./gpt_log/{create_report_file_name}') + generated_conclusion_files.append(f'{get_log_folder()}/{create_report_file_name}') chatbot.append((f"{fp}完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 @@ -261,49 +264,3 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 -class construct_html(): - def __init__(self) -> None: - self.css = """ -.row { - display: flex; - flex-wrap: wrap; -} - -.column { - flex: 1; - padding: 10px; -} - -.table-header { - font-weight: bold; - border-bottom: 1px solid black; -} - -.table-row { - border-bottom: 1px solid lightgray; -} - -.table-cell { - padding: 5px; -} - """ - self.html_string = f'翻译结果' - - - def add_row(self, a, b): - tmp = """ -
-
REPLACE_A
-
REPLACE_B
-
- """ - from toolbox import markdown_convertion - tmp = tmp.replace('REPLACE_A', markdown_convertion(a)) - tmp = tmp.replace('REPLACE_B', markdown_convertion(b)) - self.html_string += tmp - - - def save_file(self, file_name): - with open(os.path.join(get_log_folder(), file_name), 'w', encoding='utf8') as f: - f.write(self.html_string.encode('utf-8', 'ignore').decode()) - return os.path.join(get_log_folder(), file_name) diff --git a/crazy_functions/生成函数注释.py b/crazy_functions/生成函数注释.py index a564f21d..bf3da6a4 100644 --- a/crazy_functions/生成函数注释.py +++ b/crazy_functions/生成函数注释.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive fast_debug = False @@ -27,7 +28,8 @@ def 生成函数注释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, if not fast_debug: time.sleep(2) if not fast_debug: - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/虚空终端.py b/crazy_functions/虚空终端.py index 2e1b5236..5f33249e 100644 --- a/crazy_functions/虚空终端.py +++ b/crazy_functions/虚空终端.py @@ -25,11 +25,12 @@ explain_msg = """ 1. 请用**自然语言**描述您需要做什么。例如: - 「请调用插件,为我翻译PDF论文,论文我刚刚放到上传区了」 - - 「请调用插件翻译PDF论文,地址为https://aaa/bbb/ccc.pdf」 + - 「请调用插件翻译PDF论文,地址为https://openreview.net/pdf?id=rJl0r3R9KX」 - 「把Arxiv论文翻译成中文PDF,arxiv论文的ID是1812.10695,记得用插件!」 - 「生成一张图片,图中鲜花怒放,绿草如茵,用插件实现」 - 「用插件翻译README,Github网址是https://github.com/facebookresearch/co-tracker」 - 「我不喜欢当前的界面颜色,修改配置,把主题THEME更换为THEME="High-Contrast"」 + - 「请调用插件,解析python源代码项目,代码我刚刚打包拖到上传区了」 - 「请问Transformer网络的结构是怎样的?」 2. 您可以打开插件下拉菜单以了解本项目的各种能力。 @@ -45,7 +46,7 @@ explain_msg = """ from pydantic import BaseModel, Field from typing import List -from toolbox import CatchException, update_ui, gen_time_str +from toolbox import CatchException, update_ui, is_the_upload_folder from toolbox import update_ui_lastest_msg, disable_auto_promotion from request_llm.bridge_all import predict_no_ui_long_connection from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive @@ -111,7 +112,7 @@ def 虚空终端(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt # 用简单的关键词检测用户意图 is_certain, _ = analyze_intention_with_simple_rules(txt) - if txt.startswith('private_upload/') and len(txt) == 34: + if is_the_upload_folder(txt): state.set_state(chatbot=chatbot, key='has_provided_explaination', value=False) appendix_msg = "\n\n**很好,您已经上传了文件**,现在请您描述您的需求。" diff --git a/crazy_functions/解析JupyterNotebook.py b/crazy_functions/解析JupyterNotebook.py index b4bcd561..d4a3b49e 100644 --- a/crazy_functions/解析JupyterNotebook.py +++ b/crazy_functions/解析JupyterNotebook.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = True @@ -110,7 +111,8 @@ def ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # <-------- 写入文件,退出 ----------> - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/解析项目源代码.py b/crazy_functions/解析项目源代码.py index 30ae4440..10333629 100644 --- a/crazy_functions/解析项目源代码.py +++ b/crazy_functions/解析项目源代码.py @@ -1,12 +1,13 @@ -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import update_ui, promote_file_to_downloadzone, disable_auto_promotion +from toolbox import CatchException, report_execption, write_history_to_file from .crazy_utils import input_clipping def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): import os, copy from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive - msg = '正常' + disable_auto_promotion(chatbot=chatbot) + summary_batch_isolation = True inputs_array = [] inputs_show_user_array = [] @@ -22,7 +23,7 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, file_content = f.read() prefix = "接下来请你逐文件分析下面的工程" if index==0 else "" i_say = prefix + f'请对下面的程序文件做一个概述文件名是{os.path.relpath(fp, project_folder)},文件代码是 ```{file_content}```' - i_say_show_user = prefix + f'[{index}/{len(file_manifest)}] 请对下面的程序文件做一个概述: {os.path.abspath(fp)}' + i_say_show_user = prefix + f'[{index}/{len(file_manifest)}] 请对下面的程序文件做一个概述: {fp}' # 装载请求内容 inputs_array.append(i_say) inputs_show_user_array.append(i_say_show_user) @@ -43,7 +44,8 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, # 全部文件解析完成,结果写入文件,准备对工程源代码进行汇总分析 report_part_1 = copy.deepcopy(gpt_response_collection) history_to_return = report_part_1 - res = write_results_to_file(report_part_1) + res = write_history_to_file(report_part_1) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成?", "逐个文件分析已完成。" + res + "\n\n正在开始汇总。")) yield from update_ui(chatbot=chatbot, history=history_to_return) # 刷新界面 @@ -97,7 +99,8 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ############################## ################################## history_to_return.extend(report_part_2) - res = write_results_to_file(history_to_return) + res = write_history_to_file(history_to_return) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history_to_return) # 刷新界面 @@ -106,9 +109,8 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, def 解析项目本身(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): history = [] # 清空历史,以免输入溢出 import glob - file_manifest = [f for f in glob.glob('./*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + \ - [f for f in glob.glob('./crazy_functions/*.py') if ('test_project' not in f) and ('gpt_log' not in f)]+ \ - [f for f in glob.glob('./request_llm/*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + file_manifest = [f for f in glob.glob('./*.py')] + \ + [f for f in glob.glob('./*/*.py')] project_folder = './' if len(file_manifest) == 0: report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何python文件: {txt}") diff --git a/crazy_functions/读文章写摘要.py b/crazy_functions/读文章写摘要.py index 72ffe6b1..acdf632c 100644 --- a/crazy_functions/读文章写摘要.py +++ b/crazy_functions/读文章写摘要.py @@ -1,7 +1,7 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive -fast_debug = False def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): @@ -17,32 +17,29 @@ def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo chatbot.append((i_say_show_user, "[Local Message] waiting gpt response.")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - if not fast_debug: - msg = '正常' - # ** gpt request ** - gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say_show_user, llm_kwargs, chatbot, history=[], sys_prompt=system_prompt) # 带超时倒计时 - - chatbot[-1] = (i_say_show_user, gpt_say) - history.append(i_say_show_user); history.append(gpt_say) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - if not fast_debug: time.sleep(2) + msg = '正常' + gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say_show_user, llm_kwargs, chatbot, history=[], sys_prompt=system_prompt) # 带超时倒计时 + chatbot[-1] = (i_say_show_user, gpt_say) + history.append(i_say_show_user); history.append(gpt_say) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + time.sleep(2) all_file = ', '.join([os.path.relpath(fp, project_folder) for index, fp in enumerate(file_manifest)]) i_say = f'根据以上你自己的分析,对全文进行概括,用学术性语言写一段中文摘要,然后再写一段英文摘要(包括{all_file})。' chatbot.append((i_say, "[Local Message] waiting gpt response.")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - if not fast_debug: - msg = '正常' - # ** gpt request ** - gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say, llm_kwargs, chatbot, history=history, sys_prompt=system_prompt) # 带超时倒计时 + msg = '正常' + # ** gpt request ** + gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say, llm_kwargs, chatbot, history=history, sys_prompt=system_prompt) # 带超时倒计时 - chatbot[-1] = (i_say, gpt_say) - history.append(i_say); history.append(gpt_say) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - res = write_results_to_file(history) - chatbot.append(("完成了吗?", res)) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + chatbot[-1] = (i_say, gpt_say) + history.append(i_say); history.append(gpt_say) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) + chatbot.append(("完成了吗?", res)) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/辅助功能.py b/crazy_functions/辅助功能.py index e56deaa4..16854e08 100644 --- a/crazy_functions/辅助功能.py +++ b/crazy_functions/辅助功能.py @@ -2,8 +2,8 @@ # @Time : 2023/4/19 # @Author : Spike # @Descr : -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file, get_log_folder +from toolbox import update_ui, get_conf +from toolbox import CatchException from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive @@ -30,14 +30,13 @@ def 猜你想问(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt @CatchException def 清除缓存(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): - chatbot.append(['清除本地缓存数据', '执行中. 删除 gpt_log & private_upload']) + chatbot.append(['清除本地缓存数据', '执行中. 删除数据']) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 import shutil, os - gpt_log_dir = os.path.join(os.path.dirname(__file__), '..', 'gpt_log') - private_upload_dir = os.path.join(os.path.dirname(__file__), '..', 'private_upload') - shutil.rmtree(gpt_log_dir, ignore_errors=True) - shutil.rmtree(private_upload_dir, ignore_errors=True) + PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING') + shutil.rmtree(PATH_LOGGING, ignore_errors=True) + shutil.rmtree(PATH_PRIVATE_UPLOAD, ignore_errors=True) chatbot.append(['清除本地缓存数据', '执行完成']) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 \ No newline at end of file diff --git a/docs/GithubAction+AllCapacity b/docs/GithubAction+AllCapacity index f6f05daa..2ed9f4c9 100644 --- a/docs/GithubAction+AllCapacity +++ b/docs/GithubAction+AllCapacity @@ -12,7 +12,7 @@ RUN python3 -m pip install torch --extra-index-url https://download.pytorch.org/ RUN python3 -m pip install openai numpy arxiv rich RUN python3 -m pip install colorama Markdown pygments pymupdf RUN python3 -m pip install python-docx moviepy pdfminer -RUN python3 -m pip install zh_langchain==0.2.1 +RUN python3 -m pip install zh_langchain==0.2.1 pypinyin RUN python3 -m pip install nougat-ocr RUN python3 -m pip install rarfile py7zr RUN python3 -m pip install aliyun-python-sdk-core==2.13.3 pyOpenSSL scipy git+https://github.com/aliyun/alibabacloud-nls-python-sdk.git diff --git a/docs/translate_english.json b/docs/translate_english.json index 667956c8..91eba9d3 100644 --- a/docs/translate_english.json +++ b/docs/translate_english.json @@ -299,7 +299,6 @@ "地址🚀": "Address 🚀", "感谢热情的": "Thanks to the enthusiastic", "开发者们❤️": "Developers ❤️", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "All inquiry records will be automatically saved in the local directory ./gpt_log/chat_secrets.log", "请注意自我隐私保护哦!": "Please pay attention to self-privacy protection!", "当前模型": "Current model", "输入区": "Input area", @@ -892,7 +891,6 @@ "保存当前对话": "Save current conversation", "您可以调用“LoadConversationHistoryArchive”还原当下的对话": "You can call 'LoadConversationHistoryArchive' to restore the current conversation", "警告!被保存的对话历史可以被使用该系统的任何人查阅": "Warning! The saved conversation history can be viewed by anyone using this system", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT conversation history *.html", "正在查找对话历史文件": "Looking for conversation history file", "html格式": "HTML format", "找不到任何html文件": "No HTML files found", @@ -908,7 +906,6 @@ "pip install pywin32 用于doc格式": "pip install pywin32 for doc format", "仅支持Win平台": "Only supports Win platform", "打开文件": "Open file", - "private_upload里面的文件名在解压zip后容易出现乱码": "The file name in private_upload is prone to garbled characters after unzipping", "rar和7z格式正常": "RAR and 7z formats are normal", "故可以只分析文章内容": "So you can only analyze the content of the article", "不输入文件名": "Do not enter the file name", @@ -1364,7 +1361,6 @@ "注意文章中的每一句话都要翻译": "Please translate every sentence in the article", "一、论文概况": "I. Overview of the paper", "二、论文翻译": "II. Translation of the paper", - "/gpt_log/总结论文-": "/gpt_log/Summary of the paper-", "给出输出文件清单": "Provide a list of output files", "第 0 步": "Step 0", "切割PDF": "Split PDF", @@ -1564,7 +1560,6 @@ "广义速度": "Generalized velocity", "粒子的固有": "Intrinsic of particle", "一个包含所有切割音频片段文件路径的列表": "A list containing the file paths of all segmented audio clips", - "/gpt_log/翻译-": "Translation log-", "计算文件总时长和切割点": "Calculate total duration and cutting points of the file", "总结音频": "Summarize audio", "作者": "Author", @@ -2339,7 +2334,6 @@ "将文件拖动到文件上传区": "Drag and drop the file to the file upload area", "如果意图模糊": "If the intent is ambiguous", "星火认知大模型": "Spark Cognitive Big Model", - "执行中. 删除 gpt_log & private_upload": "Executing. Delete gpt_log & private_upload", "默认 Color = secondary": "Default Color = secondary", "此处也不需要修改": "No modification is needed here", "⭐ ⭐ ⭐ 分析用户意图": "⭐ ⭐ ⭐ Analyze user intent", @@ -2448,5 +2442,76 @@ "插件说明": "Plugin description", "├── CODE_HIGHLIGHT 代码高亮": "├── CODE_HIGHLIGHT Code highlighting", "记得用插件": "Remember to use the plugin", - "谨慎操作": "Handle with caution" + "谨慎操作": "Handle with caution", + "private_upload里面的文件名在解压zip后容易出现乱码": "The file name inside private_upload is prone to garbled characters after unzipping", + "直接返回报错": "Direct return error", + "临时的上传文件夹位置": "Temporary upload folder location", + "使用latex格式 测试3 写出麦克斯韦方程组": "Write Maxwell's equations using latex format for test 3", + "这是一张图片": "This is an image", + "没有发现任何近期上传的文件": "No recent uploaded files found", + "如url未成功匹配返回None": "Return None if the URL does not match successfully", + "如果有Latex环境": "If there is a Latex environment", + "第一次运行时": "When running for the first time", + "创建工作路径": "Create a working directory", + "向": "To", + "执行中. 删除数据": "Executing. Deleting data", + "CodeInterpreter开源版": "CodeInterpreter open source version", + "建议选择更稳定的接口": "It is recommended to choose a more stable interface", + "现在您点击任意函数插件时": "Now when you click on any function plugin", + "请使用“LatexEnglishCorrection+高亮”插件": "Please use the 'LatexEnglishCorrection+Highlight' plugin", + "安装完成": "Installation completed", + "记得用插件!」": "Remember to use the plugin!", + "结论": "Conclusion", + "无法下载资源": "Unable to download resources", + "首先排除一个one-api没有done数据包的第三方Bug情形": "First exclude a third-party bug where one-api does not have a done data package", + "知识库中添加文件": "Add files to the knowledge base", + "处理重名的章节": "Handling duplicate chapter names", + "先上传文件素材": "Upload file materials first", + "无法从google获取信息!": "Unable to retrieve information from Google!", + "展示如下": "Display as follows", + "「把Arxiv论文翻译成中文PDF": "Translate Arxiv papers into Chinese PDF", + "论文我刚刚放到上传区了」": "I just put the paper in the upload area", + "正在下载Gradio主题": "Downloading Gradio themes", + "再运行此插件": "Run this plugin again", + "记录近期文件": "Record recent files", + "粗心检查": "Careful check", + "更多主题": "More themes", + "//huggingface.co/spaces/gradio/theme-gallery 可选": "//huggingface.co/spaces/gradio/theme-gallery optional", + "由 test_on_result_chg": "By test_on_result_chg", + "所有问询记录将自动保存在本地目录./": "All inquiry records will be automatically saved in the local directory ./", + "正在解析论文": "Analyzing the paper", + "逐个文件转移到目标路径": "Move each file to the target path", + "最多重试5次": "Retry up to 5 times", + "日志文件夹的位置": "Location of the log folder", + "我们暂时无法解析此PDF文档": "We are temporarily unable to parse this PDF document", + "文件检索": "File retrieval", + "/**/chatGPT对话历史*.html": "/**/chatGPT conversation history*.html", + "非OpenAI官方接口返回了错误": "Non-OpenAI official interface returned an error", + "如果在Arxiv上匹配失败": "If the match fails on Arxiv", + "文件进入知识库后可长期保存": "Files can be saved for a long time after entering the knowledge base", + "您可以再次重试": "You can try again", + "整理文件集合": "Organize file collection", + "检测到有缺陷的非OpenAI官方接口": "Detected defective non-OpenAI official interface", + "此插件不调用Latex": "This plugin does not call Latex", + "移除过时的旧文件从而节省空间&保护隐私": "Remove outdated old files to save space & protect privacy", + "代码我刚刚打包拖到上传区了」": "I just packed the code and dragged it to the upload area", + "将图像转为灰度图像": "Convert the image to grayscale", + "待排除": "To be excluded", + "请勿修改": "Please do not modify", + "crazy_functions/代码重写为全英文_多线程.py": "crazy_functions/code rewritten to all English_multi-threading.py", + "开发中": "Under development", + "请查阅Gradio主题商店": "Please refer to the Gradio theme store", + "输出消息": "Output message", + "其他情况": "Other situations", + "获取文献失败": "Failed to retrieve literature", + "可以通过再次调用本插件的方式": "You can use this plugin again by calling it", + "保留下半部分": "Keep the lower half", + "排除问题": "Exclude the problem", + "知识库": "Knowledge base", + "ParsePDF失败": "ParsePDF failed", + "向知识库追加更多文档": "Append more documents to the knowledge base", + "此处待注入的知识库名称id": "The knowledge base name ID to be injected here", + "您需要构建知识库后再运行此插件": "You need to build the knowledge base before running this plugin", + "判定是否为公式 | 测试1 写出洛伦兹定律": "Determine whether it is a formula | Test 1 write out the Lorentz law", + "构建知识库后": "After building the knowledge base" } \ No newline at end of file diff --git a/docs/translate_japanese.json b/docs/translate_japanese.json index 15a4b319..ecd370e9 100644 --- a/docs/translate_japanese.json +++ b/docs/translate_japanese.json @@ -301,7 +301,6 @@ "缺少的依赖": "不足している依存関係", "紫色": "紫色", "唤起高级参数输入区": "高度なパラメータ入力エリアを呼び出す", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "すべての問い合わせ記録は自動的にローカルディレクトリ./gpt_log/chat_secrets.logに保存されます", "则换行符更有可能表示段落分隔": "したがって、改行記号は段落の区切りを表す可能性がより高いです", ";4、引用数量": ";4、引用数量", "中转网址预览": "中継ウェブサイトのプレビュー", @@ -448,7 +447,6 @@ "表示函数是否成功执行": "関数が正常に実行されたかどうかを示す", "一般原样传递下去就行": "通常はそのまま渡すだけでよい", "琥珀色": "琥珀色", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT対話履歴*.html", "jittorllms 没有 sys_prompt 接口": "jittorllmsにはsys_promptインターフェースがありません", "清除": "クリア", "小于正文的": "本文より小さい", @@ -1234,7 +1232,6 @@ "找不到任何前端相关文件": "No frontend-related files can be found", "Not enough point. API2D账户点数不足": "Not enough points. API2D account points are insufficient", "当前版本": "Current version", - "/gpt_log/总结论文-": "/gpt_log/Summary paper-", "1. 临时解决方案": "1. Temporary solution", "第8步": "Step 8", "历史": "History", diff --git a/docs/translate_std.json b/docs/translate_std.json index 63b6baa4..84690c2e 100644 --- a/docs/translate_std.json +++ b/docs/translate_std.json @@ -88,5 +88,7 @@ "辅助功能": "Accessibility", "虚空终端": "VoidTerminal", "解析PDF_基于GROBID": "ParsePDF_BasedOnGROBID", - "虚空终端主路由": "VoidTerminalMainRoute" + "虚空终端主路由": "VoidTerminalMainRoute", + "批量翻译PDF文档_NOUGAT": "BatchTranslatePDFDocuments_NOUGAT", + "解析PDF_基于NOUGAT": "ParsePDF_NOUGAT" } \ No newline at end of file diff --git a/docs/translate_traditionalchinese.json b/docs/translate_traditionalchinese.json index cf12de1c..efaa62fc 100644 --- a/docs/translate_traditionalchinese.json +++ b/docs/translate_traditionalchinese.json @@ -314,7 +314,6 @@ "请用markdown格式输出": "請用 Markdown 格式輸出", "模仿ChatPDF": "模仿 ChatPDF", "等待多久判定为超时": "等待多久判定為超時", - "/gpt_log/总结论文-": "/gpt_log/總結論文-", "请结合互联网信息回答以下问题": "請結合互聯網信息回答以下問題", "IP查询频率受限": "IP查詢頻率受限", "高级参数输入区的显示提示": "高級參數輸入區的顯示提示", @@ -511,7 +510,6 @@ "將生成的報告自動投射到文件上傳區": "將生成的報告自動上傳到文件區", "函數插件作者": "函數插件作者", "將要匹配的模式": "將要匹配的模式", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "所有詢問記錄將自動保存在本地目錄./gpt_log/chat_secrets.log", "正在分析一个项目的源代码": "正在分析一個專案的源代碼", "使每个段落之间有两个换行符分隔": "使每個段落之間有兩個換行符分隔", "并在被装饰的函数上执行": "並在被裝飾的函數上執行", @@ -1059,7 +1057,6 @@ "重试中": "重試中", "月": "月份", "localhost意思是代理软件安装在本机上": "localhost意思是代理軟體安裝在本機上", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT對話歷史*.html", "的长度必须小于 2500 个 Token": "長度必須小於 2500 個 Token", "抽取可用的api-key": "提取可用的api-key", "增强报告的可读性": "增強報告的可讀性", diff --git a/multi_language.py b/multi_language.py index c4ed36eb..8e3ac9d7 100644 --- a/multi_language.py +++ b/multi_language.py @@ -33,9 +33,11 @@ import functools import re import pickle import time +from toolbox import get_conf -CACHE_FOLDER = "gpt_log" -blacklist = ['multi-language', 'gpt_log', '.git', 'private_upload', 'multi_language.py', 'build', '.github', '.vscode', '__pycache__', 'venv'] +CACHE_FOLDER, = get_conf('PATH_LOGGING') + +blacklist = ['multi-language', CACHE_FOLDER, '.git', 'private_upload', 'multi_language.py', 'build', '.github', '.vscode', '__pycache__', 'venv'] # LANG = "TraditionalChinese" # TransPrompt = f"Replace each json value `#` with translated results in Traditional Chinese, e.g., \"原始文本\":\"翻譯後文字\". Keep Json format. Do not answer #." diff --git a/request_llm/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py index 929a7546..a1b6ba47 100644 --- a/request_llm/bridge_chatgpt.py +++ b/request_llm/bridge_chatgpt.py @@ -21,7 +21,7 @@ import importlib # config_private.py放自己的秘密如API和代理网址 # 读取时首先看是否存在私密的config_private配置文件(不受git管控),如果有,则覆盖原config文件 -from toolbox import get_conf, update_ui, is_any_api_key, select_api_key, what_keys, clip_history, trimmed_format_exc +from toolbox import get_conf, update_ui, is_any_api_key, select_api_key, what_keys, clip_history, trimmed_format_exc, is_the_upload_folder proxies, TIMEOUT_SECONDS, MAX_RETRY, API_ORG = \ get_conf('proxies', 'TIMEOUT_SECONDS', 'MAX_RETRY', 'API_ORG') @@ -72,6 +72,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", stream_response = response.iter_lines() result = '' + json_data = None while True: try: chunk = next(stream_response).decode() except StopIteration: @@ -90,20 +91,21 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", delta = json_data["delta"] if len(delta) == 0: break if "role" in delta: continue - if "content" in delta: + if "content" in delta: result += delta["content"] if not console_slience: print(delta["content"], end='') if observe_window is not None: # 观测窗,把已经获取的数据显示出去 - if len(observe_window) >= 1: observe_window[0] += delta["content"] + if len(observe_window) >= 1: + observe_window[0] += delta["content"] # 看门狗,如果超过期限没有喂狗,则终止 - if len(observe_window) >= 2: + if len(observe_window) >= 2: if (time.time()-observe_window[1]) > watch_dog_patience: raise RuntimeError("用户取消了程序。") else: raise RuntimeError("意外Json结构:"+delta) - if json_data['finish_reason'] == 'content_filter': + if json_data and json_data['finish_reason'] == 'content_filter': raise RuntimeError("由于提问含不合规内容被Azure过滤。") - if json_data['finish_reason'] == 'length': + if json_data and json_data['finish_reason'] == 'length': raise ConnectionAbortedError("正常结束,但显示Token不足,导致输出不完整,请削减单次输入的文本量。") return result @@ -128,6 +130,7 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp yield from update_ui(chatbot=chatbot, history=history, msg="缺少api_key") # 刷新界面 return + user_input = inputs if additional_fn is not None: from core_functional import handle_core_functionality inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot) @@ -138,8 +141,8 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面 # check mis-behavior - if raw_input.startswith('private_upload/') and len(raw_input) == 34: - chatbot[-1] = (inputs, f"[Local Message] 检测到操作错误!当您上传文档之后,需要点击“函数插件区”按钮进行处理,而不是点击“提交”按钮。") + if is_the_upload_folder(user_input): + chatbot[-1] = (inputs, f"[Local Message] 检测到操作错误!当您上传文档之后,需点击“**函数插件区**”按钮进行处理,请勿点击“提交”按钮或者“基础功能区”按钮。") yield from update_ui(chatbot=chatbot, history=history, msg="正常") # 刷新界面 time.sleep(2) @@ -179,8 +182,13 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp # 非OpenAI官方接口的出现这样的报错,OpenAI和API2D不会走这里 chunk_decoded = chunk.decode() error_msg = chunk_decoded + # 首先排除一个one-api没有done数据包的第三方Bug情形 + if len(gpt_replying_buffer.strip()) > 0 and len(error_msg) == 0: + yield from update_ui(chatbot=chatbot, history=history, msg="检测到有缺陷的非OpenAI官方接口,建议选择更稳定的接口。") + break + # 其他情况,直接返回报错 chatbot, history = handle_error(inputs, llm_kwargs, chatbot, history, chunk_decoded, error_msg) - yield from update_ui(chatbot=chatbot, history=history, msg="非Openai官方接口返回了错误:" + chunk.decode()) # 刷新界面 + yield from update_ui(chatbot=chatbot, history=history, msg="非OpenAI官方接口返回了错误:" + chunk.decode()) # 刷新界面 return chunk_decoded = chunk.decode() @@ -199,7 +207,7 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp chunkjson = json.loads(chunk_decoded[6:]) status_text = f"finish_reason: {chunkjson['choices'][0].get('finish_reason', 'null')}" # 如果这里抛出异常,一般是文本过长,详情见get_full_error的输出 - gpt_replying_buffer = gpt_replying_buffer + json.loads(chunk_decoded[6:])['choices'][0]["delta"]["content"] + gpt_replying_buffer = gpt_replying_buffer + chunkjson['choices'][0]["delta"]["content"] history[-1] = gpt_replying_buffer chatbot[-1] = (history[-2], history[-1]) yield from update_ui(chatbot=chatbot, history=history, msg=status_text) # 刷新界面 diff --git a/request_llm/com_sparkapi.py b/request_llm/com_sparkapi.py index 0b8d655d..ae970b9a 100644 --- a/request_llm/com_sparkapi.py +++ b/request_llm/com_sparkapi.py @@ -109,6 +109,7 @@ class SparkRequestInstance(): code = data['header']['code'] if code != 0: print(f'请求错误: {code}, {data}') + self.result_buf += str(data) ws.close() self.time_to_exit_event.set() else: diff --git a/themes/gradios.py b/themes/gradios.py new file mode 100644 index 00000000..8b661a56 --- /dev/null +++ b/themes/gradios.py @@ -0,0 +1,46 @@ +import gradio as gr +import logging +from toolbox import get_conf, ProxyNetworkActivate +CODE_HIGHLIGHT, ADD_WAIFU, LAYOUT = get_conf('CODE_HIGHLIGHT', 'ADD_WAIFU', 'LAYOUT') + +def adjust_theme(): + + try: + set_theme = gr.themes.ThemeClass() + with ProxyNetworkActivate(): + logging.info('正在下载Gradio主题,请稍等。') + THEME, = get_conf('THEME') + if THEME.startswith('Huggingface-'): THEME = THEME.lstrip('Huggingface-') + if THEME.startswith('huggingface-'): THEME = THEME.lstrip('huggingface-') + set_theme = set_theme.from_hub(THEME.lower()) + + if LAYOUT=="TOP-DOWN": + js = "" + else: + with open('themes/common.js', 'r', encoding='utf8') as f: + js = f"" + + # 添加一个萌萌的看板娘 + if ADD_WAIFU: + js += """ + + + + """ + gradio_original_template_fn = gr.routes.templates.TemplateResponse + def gradio_new_template_fn(*args, **kwargs): + res = gradio_original_template_fn(*args, **kwargs) + res.body = res.body.replace(b'', f'{js}'.encode("utf8")) + res.init_headers() + return res + gr.routes.templates.TemplateResponse = gradio_new_template_fn # override gradio template + except Exception as e: + set_theme = None + from toolbox import trimmed_format_exc + logging.error('gradio版本较旧, 不能自定义字体和颜色:', trimmed_format_exc()) + return set_theme + +# with open("themes/default.css", "r", encoding="utf-8") as f: +# advanced_css = f.read() +with open("themes/common.css", "r", encoding="utf-8") as f: + advanced_css = f.read() diff --git a/themes/theme.py b/themes/theme.py index 5cba541c..dbb8f1e2 100644 --- a/themes/theme.py +++ b/themes/theme.py @@ -8,6 +8,9 @@ if THEME == 'Chuanhu-Small-and-Beautiful': elif THEME == 'High-Contrast': from .contrast import adjust_theme, advanced_css theme_declaration = "" +elif '/' in THEME: + from .gradios import adjust_theme, advanced_css + theme_declaration = "" else: from .default import adjust_theme, advanced_css theme_declaration = "" diff --git a/toolbox.py b/toolbox.py index ab8d046a..1452c13a 100644 --- a/toolbox.py +++ b/toolbox.py @@ -5,6 +5,8 @@ import inspect import re import os import gradio +import shutil +import glob from latex2mathml.converter import convert as tex2mathml from functools import wraps, lru_cache pj = os.path.join @@ -77,14 +79,24 @@ def ArgsGeneralWrapper(f): } chatbot_with_cookie = ChatBotWithCookies(cookies) chatbot_with_cookie.write_list(chatbot) + if cookies.get('lock_plugin', None) is None: # 正常状态 - yield from f(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, *args) + if len(args) == 0: # 插件通道 + yield from f(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, request) + else: # 对话通道,或者基础功能通道 + yield from f(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, *args) else: - # 处理个别特殊插件的锁定状态 + # 处理少数情况下的特殊插件的锁定状态 module, fn_name = cookies['lock_plugin'].split('->') f_hot_reload = getattr(importlib.import_module(module, fn_name), fn_name) yield from f_hot_reload(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, request) + # 判断一下用户是否错误地通过对话通道进入,如果是,则进行提醒 + final_cookies = chatbot_with_cookie.get_cookies() + # len(args) != 0 代表“提交”键对话通道,或者基础功能通道 + if len(args) != 0 and 'files_to_promote' in final_cookies and len(final_cookies['files_to_promote']) > 0: + chatbot_with_cookie.append(["检测到**滞留的缓存文档**,请及时处理。", "请及时点击“**保存当前对话**”获取所有滞留文档。"]) + yield from update_ui(chatbot_with_cookie, final_cookies['history'], msg="检测到被滞留的缓存文档") return decorated @@ -94,7 +106,8 @@ def update_ui(chatbot, history, msg='正常', **kwargs): # 刷新界面 """ assert isinstance(chatbot, ChatBotWithCookies), "在传递chatbot的过程中不要将其丢弃。必要时, 可用clear将其清空, 然后用for+append循环重新赋值。" cookies = chatbot.get_cookies() - + # 备份一份History作为记录 + cookies.update({'history': history}) # 解决插件锁定时的界面显示问题 if cookies.get('lock_plugin', None): label = cookies.get('llm_model', "") + " | " + "正在锁定插件" + cookies.get('lock_plugin', None) @@ -171,7 +184,7 @@ def HotReload(f): ======================================================================== 第二部分 其他小工具: - - write_results_to_file: 将结果写入markdown文件中 + - write_history_to_file: 将结果写入markdown文件中 - regular_txt_to_markdown: 将普通文本转换为Markdown格式的文本。 - report_execption: 向chatbot中添加简单的意外错误信息 - text_divide_paragraph: 将文本按照段落分隔符分割开,生成带有段落标签的HTML代码。 @@ -203,36 +216,6 @@ def get_reduce_token_percent(text): return 0.5, '不详' -def write_results_to_file(history, file_name=None): - """ - 将对话记录history以Markdown格式写入文件中。如果没有指定文件名,则使用当前时间生成文件名。 - """ - import os - import time - if file_name is None: - # file_name = time.strftime("chatGPT分析报告%Y-%m-%d-%H-%M-%S", time.localtime()) + '.md' - file_name = 'GPT-Report-' + gen_time_str() + '.md' - os.makedirs('./gpt_log/', exist_ok=True) - with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: - f.write('# GPT-Academic Report\n') - for i, content in enumerate(history): - try: - if type(content) != str: content = str(content) - except: - continue - if i % 2 == 0: - f.write('## ') - try: - f.write(content) - except: - # remove everything that cannot be handled by utf8 - f.write(content.encode('utf-8', 'ignore').decode()) - f.write('\n\n') - res = '以上材料已经被写入:\t' + os.path.abspath(f'./gpt_log/{file_name}') - print(res) - return res - - def write_history_to_file(history, file_basename=None, file_fullname=None): """ 将对话记录history以Markdown格式写入文件中。如果没有指定文件名,则使用当前时间生成文件名。 @@ -241,9 +224,9 @@ def write_history_to_file(history, file_basename=None, file_fullname=None): import time if file_fullname is None: if file_basename is not None: - file_fullname = os.path.join(get_log_folder(), file_basename) + file_fullname = pj(get_log_folder(), file_basename) else: - file_fullname = os.path.join(get_log_folder(), f'GPT-Academic-{gen_time_str()}.md') + file_fullname = pj(get_log_folder(), f'GPT-Academic-{gen_time_str()}.md') os.makedirs(os.path.dirname(file_fullname), exist_ok=True) with open(file_fullname, 'w', encoding='utf8') as f: f.write('# GPT-Academic Report\n') @@ -519,7 +502,7 @@ def find_recent_files(directory): if not os.path.exists(directory): os.makedirs(directory, exist_ok=True) for filename in os.listdir(directory): - file_path = os.path.join(directory, filename) + file_path = pj(directory, filename) if file_path.endswith('.log'): continue created_time = os.path.getmtime(file_path) @@ -534,7 +517,7 @@ def promote_file_to_downloadzone(file, rename_file=None, chatbot=None): # 将文件复制一份到下载区 import shutil if rename_file is None: rename_file = f'{gen_time_str()}-{os.path.basename(file)}' - new_path = os.path.join(get_log_folder(), rename_file) + new_path = pj(get_log_folder(), rename_file) # 如果已经存在,先删除 if os.path.exists(new_path) and not os.path.samefile(new_path, file): os.remove(new_path) # 把文件复制过去 @@ -549,44 +532,70 @@ def disable_auto_promotion(chatbot): chatbot._cookies.update({'files_to_promote': []}) return -def on_file_uploaded(files, chatbot, txt, txt2, checkboxes, cookies): +def is_the_upload_folder(string): + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + pattern = r'^PATH_PRIVATE_UPLOAD/[A-Za-z0-9_-]+/\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}$' + pattern = pattern.replace('PATH_PRIVATE_UPLOAD', PATH_PRIVATE_UPLOAD) + if re.match(pattern, string): return True + else: return False + +def del_outdated_uploads(outdate_time_seconds): + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + current_time = time.time() + one_hour_ago = current_time - outdate_time_seconds + # Get a list of all subdirectories in the PATH_PRIVATE_UPLOAD folder + # Remove subdirectories that are older than one hour + for subdirectory in glob.glob(f'{PATH_PRIVATE_UPLOAD}/*/*'): + subdirectory_time = os.path.getmtime(subdirectory) + if subdirectory_time < one_hour_ago: + try: shutil.rmtree(subdirectory) + except: pass + return + +def on_file_uploaded(request: gradio.Request, files, chatbot, txt, txt2, checkboxes, cookies): """ 当文件被上传时的回调函数 """ if len(files) == 0: return chatbot, txt - import shutil - import os - import time - import glob - from toolbox import extract_archive - try: - shutil.rmtree('./private_upload/') - except: - pass + + # 移除过时的旧文件从而节省空间&保护隐私 + outdate_time_seconds = 60 + del_outdated_uploads(outdate_time_seconds) + + # 创建工作路径 + user_name = "default" if not request.username else request.username time_tag = gen_time_str() - os.makedirs(f'private_upload/{time_tag}', exist_ok=True) - err_msg = '' + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + target_path_base = pj(PATH_PRIVATE_UPLOAD, user_name, time_tag) + os.makedirs(target_path_base, exist_ok=True) + + # 逐个文件转移到目标路径 + upload_msg = '' for file in files: file_origin_name = os.path.basename(file.orig_name) - shutil.copy(file.name, f'private_upload/{time_tag}/{file_origin_name}') - err_msg += extract_archive(f'private_upload/{time_tag}/{file_origin_name}', - dest_dir=f'private_upload/{time_tag}/{file_origin_name}.extract') - moved_files = [fp for fp in glob.glob('private_upload/**/*', recursive=True)] - if "底部输入区" in checkboxes: - txt = "" - txt2 = f'private_upload/{time_tag}' + this_file_path = pj(target_path_base, file_origin_name) + shutil.move(file.name, this_file_path) + upload_msg += extract_archive(file_path=this_file_path, dest_dir=this_file_path+'.extract') + + # 整理文件集合 + moved_files = [fp for fp in glob.glob(f'{target_path_base}/**/*', recursive=True)] + if "底部输入区" in checkboxes: + txt, txt2 = "", target_path_base else: - txt = f'private_upload/{time_tag}' - txt2 = "" + txt, txt2 = target_path_base, "" + + # 输出消息 moved_files_str = '\t\n\n'.join(moved_files) - chatbot.append(['我上传了文件,请查收', + chatbot.append(['我上传了文件,请查收', f'[Local Message] 收到以下文件: \n\n{moved_files_str}' + f'\n\n调用路径参数已自动修正到: \n\n{txt}' + - f'\n\n现在您点击任意函数插件时,以上文件将被作为输入参数'+err_msg]) + f'\n\n现在您点击任意函数插件时,以上文件将被作为输入参数'+upload_msg]) + + # 记录近期文件 cookies.update({ 'most_recent_uploaded': { - 'path': f'private_upload/{time_tag}', + 'path': target_path_base, 'time': time.time(), 'time_str': time_tag }}) @@ -595,11 +604,12 @@ def on_file_uploaded(files, chatbot, txt, txt2, checkboxes, cookies): def on_report_generated(cookies, files, chatbot): from toolbox import find_recent_files + PATH_LOGGING, = get_conf('PATH_LOGGING') if 'files_to_promote' in cookies: report_files = cookies['files_to_promote'] cookies.pop('files_to_promote') else: - report_files = find_recent_files('gpt_log') + report_files = find_recent_files(PATH_LOGGING) if len(report_files) == 0: return cookies, None, chatbot # files.extend(report_files) @@ -909,34 +919,35 @@ def zip_folder(source_folder, dest_folder, zip_name): return # Create the name for the zip file - zip_file = os.path.join(dest_folder, zip_name) + zip_file = pj(dest_folder, zip_name) # Create a ZipFile object with zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED) as zipf: # Walk through the source folder and add files to the zip file for foldername, subfolders, filenames in os.walk(source_folder): for filename in filenames: - filepath = os.path.join(foldername, filename) + filepath = pj(foldername, filename) zipf.write(filepath, arcname=os.path.relpath(filepath, source_folder)) # Move the zip file to the destination folder (if it wasn't already there) if os.path.dirname(zip_file) != dest_folder: - os.rename(zip_file, os.path.join(dest_folder, os.path.basename(zip_file))) - zip_file = os.path.join(dest_folder, os.path.basename(zip_file)) + os.rename(zip_file, pj(dest_folder, os.path.basename(zip_file))) + zip_file = pj(dest_folder, os.path.basename(zip_file)) print(f"Zip file created at {zip_file}") def zip_result(folder): t = gen_time_str() - zip_folder(folder, './gpt_log/', f'{t}-result.zip') - return pj('./gpt_log/', f'{t}-result.zip') + zip_folder(folder, get_log_folder(), f'{t}-result.zip') + return pj(get_log_folder(), f'{t}-result.zip') def gen_time_str(): import time return time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) def get_log_folder(user='default', plugin_name='shared'): - _dir = os.path.join(os.path.dirname(__file__), 'gpt_log', user, plugin_name) + PATH_LOGGING, = get_conf('PATH_LOGGING') + _dir = pj(PATH_LOGGING, user, plugin_name) if not os.path.exists(_dir): os.makedirs(_dir) return _dir diff --git a/version b/version index 369a8008..897b0428 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ { - "version": 3.50, + "version": 3.52, "show_feature": true, - "new_feature": "支持插件分类! <-> 支持用户使用自然语言调度各个插件(虚空终端) ! <-> 改进UI,设计新主题 <-> 支持借助GROBID实现PDF高精度翻译 <-> 接入百度千帆平台和文心一言 <-> 接入阿里通义千问、讯飞星火、上海AI-Lab书生 <-> 优化一键升级 <-> 提高arxiv翻译速度和成功率" + "new_feature": "提高稳定性&解决多用户冲突问题 <-> 支持插件分类和更多UI皮肤外观 <-> 支持用户使用自然语言调度各个插件(虚空终端) ! <-> 改进UI,设计新主题 <-> 支持借助GROBID实现PDF高精度翻译 <-> 接入百度千帆平台和文心一言 <-> 接入阿里通义千问、讯飞星火、上海AI-Lab书生 <-> 优化一键升级 <-> 提高arxiv翻译速度和成功率" }