Merge Latest Frontier (#1991)
* logging sys to loguru: stage 1 complete * import loguru: stage 2 * logging -> loguru: stage 3 * support o1-preview and o1-mini * logging -> loguru stage 4 * update social helper * logging -> loguru: final stage * fix: console output * update translation matrix * fix: loguru argument error with proxy enabled (#1977) * relax llama index version * remove comment * Added some modules to support openrouter (#1975) * Added some modules for supporting openrouter model Added some modules for supporting openrouter model * Update config.py * Update .gitignore * Update bridge_openrouter.py * Not changed actually * Refactor logging in bridge_openrouter.py --------- Co-authored-by: binary-husky <qingxu.fu@outlook.com> * remove logging extra --------- Co-authored-by: Steven Moder <java20131114@gmail.com> Co-authored-by: Ren Lifei <2602264455@qq.com>
This commit is contained in:
binary-husky
@@ -2,6 +2,8 @@ import markdown
|
||||
import re
|
||||
import os
|
||||
import math
|
||||
|
||||
from loguru import logger
|
||||
from textwrap import dedent
|
||||
from functools import lru_cache
|
||||
from pymdownx.superfences import fence_code_format
|
||||
@@ -227,14 +229,14 @@ def fix_dollar_sticking_bug(txt):
|
||||
|
||||
if single_stack_height > 0:
|
||||
if txt[:(index+1)].find('\n') > 0 or txt[:(index+1)].find('<td>') > 0 or txt[:(index+1)].find('</td>') > 0:
|
||||
print('公式之中出现了异常 (Unexpect element in equation)')
|
||||
logger.error('公式之中出现了异常 (Unexpect element in equation)')
|
||||
single_stack_height = 0
|
||||
txt_result += ' $'
|
||||
continue
|
||||
|
||||
if double_stack_height > 0:
|
||||
if txt[:(index+1)].find('\n\n') > 0:
|
||||
print('公式之中出现了异常 (Unexpect element in equation)')
|
||||
logger.error('公式之中出现了异常 (Unexpect element in equation)')
|
||||
double_stack_height = 0
|
||||
txt_result += '$$'
|
||||
continue
|
||||
@@ -253,13 +255,13 @@ def fix_dollar_sticking_bug(txt):
|
||||
txt = txt[(index+2):]
|
||||
else:
|
||||
if double_stack_height != 0:
|
||||
# print(txt[:(index)])
|
||||
print('发现异常嵌套公式')
|
||||
# logger.info(txt[:(index)])
|
||||
logger.info('发现异常嵌套公式')
|
||||
if single_stack_height == 0:
|
||||
single_stack_height = 1
|
||||
else:
|
||||
single_stack_height = 0
|
||||
# print(txt[:(index)])
|
||||
# logger.info(txt[:(index)])
|
||||
txt_result += txt[:(index+1)]
|
||||
txt = txt[(index+1):]
|
||||
break
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import platform
|
||||
from sys import stdout
|
||||
from loguru import logger
|
||||
|
||||
if platform.system()=="Linux":
|
||||
pass
|
||||
@@ -59,3 +60,29 @@ def sprint亮紫(*kw):
|
||||
return "\033[1;35m"+' '.join(kw)+"\033[0m"
|
||||
def sprint亮靛(*kw):
|
||||
return "\033[1;36m"+' '.join(kw)+"\033[0m"
|
||||
|
||||
def log红(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint红(*kw))
|
||||
def log绿(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint绿(*kw))
|
||||
def log黄(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint黄(*kw))
|
||||
def log蓝(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint蓝(*kw))
|
||||
def log紫(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint紫(*kw))
|
||||
def log靛(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint靛(*kw))
|
||||
|
||||
def log亮红(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint亮红(*kw))
|
||||
def log亮绿(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint亮绿(*kw))
|
||||
def log亮黄(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint亮黄(*kw))
|
||||
def log亮蓝(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint亮蓝(*kw))
|
||||
def log亮紫(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint亮紫(*kw))
|
||||
def log亮靛(*kw,**kargs):
|
||||
logger.opt(depth=1).info(sprint亮靛(*kw))
|
||||
@@ -2,7 +2,7 @@ import importlib
|
||||
import time
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from shared_utils.colorful import print亮红, print亮绿, print亮蓝
|
||||
from shared_utils.colorful import log亮红, log亮绿, log亮蓝
|
||||
|
||||
pj = os.path.join
|
||||
default_user_name = 'default_user'
|
||||
@@ -30,13 +30,13 @@ def read_env_variable(arg, default_value):
|
||||
env_arg = os.environ[arg]
|
||||
else:
|
||||
raise KeyError
|
||||
print(f"[ENV_VAR] 尝试加载{arg}，默认值：{default_value} --> 修正值：{env_arg}")
|
||||
log亮绿(f"[ENV_VAR] 尝试加载{arg}，默认值：{default_value} --> 修正值：{env_arg}")
|
||||
try:
|
||||
if isinstance(default_value, bool):
|
||||
env_arg = env_arg.strip()
|
||||
if env_arg == 'True': r = True
|
||||
elif env_arg == 'False': r = False
|
||||
else: print('Enter True or False, but have:', env_arg); r = default_value
|
||||
else: log亮红('Expect `True` or `False`, but have:', env_arg); r = default_value
|
||||
elif isinstance(default_value, int):
|
||||
r = int(env_arg)
|
||||
elif isinstance(default_value, float):
|
||||
@@ -51,13 +51,13 @@ def read_env_variable(arg, default_value):
|
||||
assert arg == "proxies"
|
||||
r = eval(env_arg)
|
||||
else:
|
||||
print亮红(f"[ENV_VAR] 环境变量{arg}不支持通过环境变量设置! ")
|
||||
log亮红(f"[ENV_VAR] 环境变量{arg}不支持通过环境变量设置! ")
|
||||
raise KeyError
|
||||
except:
|
||||
print亮红(f"[ENV_VAR] 环境变量{arg}加载失败! ")
|
||||
log亮红(f"[ENV_VAR] 环境变量{arg}加载失败! ")
|
||||
raise KeyError(f"[ENV_VAR] 环境变量{arg}加载失败! ")
|
||||
|
||||
print亮绿(f"[ENV_VAR] 成功读取环境变量{arg}")
|
||||
log亮绿(f"[ENV_VAR] 成功读取环境变量{arg}")
|
||||
return r
|
||||
|
||||
|
||||
@@ -80,21 +80,21 @@ def read_single_conf_with_lru_cache(arg):
|
||||
if arg == 'API_URL_REDIRECT':
|
||||
oai_rd = r.get("https://api.openai.com/v1/chat/completions", None) # API_URL_REDIRECT填写格式是错误的，请阅读`https://github.com/binary-husky/gpt_academic/wiki/项目配置说明`
|
||||
if oai_rd and not oai_rd.endswith('/completions'):
|
||||
print亮红("\n\n[API_URL_REDIRECT] API_URL_REDIRECT填错了。请阅读`https://github.com/binary-husky/gpt_academic/wiki/项目配置说明`。如果您确信自己没填错，无视此消息即可。")
|
||||
log亮红("\n\n[API_URL_REDIRECT] API_URL_REDIRECT填错了。请阅读`https://github.com/binary-husky/gpt_academic/wiki/项目配置说明`。如果您确信自己没填错，无视此消息即可。")
|
||||
time.sleep(5)
|
||||
if arg == 'API_KEY':
|
||||
print亮蓝(f"[API_KEY] 本项目现已支持OpenAI和Azure的api-key。也支持同时填写多个api-key，如API_KEY=\"openai-key1,openai-key2,azure-key3\"")
|
||||
print亮蓝(f"[API_KEY] 您既可以在config.py中修改api-key(s)，也可以在问题输入区输入临时的api-key(s)，然后回车键提交后即可生效。")
|
||||
log亮蓝(f"[API_KEY] 本项目现已支持OpenAI和Azure的api-key。也支持同时填写多个api-key，如API_KEY=\"openai-key1,openai-key2,azure-key3\"")
|
||||
log亮蓝(f"[API_KEY] 您既可以在config.py中修改api-key(s)，也可以在问题输入区输入临时的api-key(s)，然后回车键提交后即可生效。")
|
||||
if is_any_api_key(r):
|
||||
print亮绿(f"[API_KEY] 您的 API_KEY 是: {r[:15]}*** API_KEY 导入成功")
|
||||
log亮绿(f"[API_KEY] 您的 API_KEY 是: {r[:15]}*** API_KEY 导入成功")
|
||||
else:
|
||||
print亮红(f"[API_KEY] 您的 API_KEY（{r[:15]}***）不满足任何一种已知的密钥格式，请在config文件中修改API密钥之后再运行（详见`https://github.com/binary-husky/gpt_academic/wiki/api_key`）。")
|
||||
log亮红(f"[API_KEY] 您的 API_KEY（{r[:15]}***）不满足任何一种已知的密钥格式，请在config文件中修改API密钥之后再运行（详见`https://github.com/binary-husky/gpt_academic/wiki/api_key`）。")
|
||||
if arg == 'proxies':
|
||||
if not read_single_conf_with_lru_cache('USE_PROXY'): r = None # 检查USE_PROXY，防止proxies单独起作用
|
||||
if r is None:
|
||||
print亮红('[PROXY] 网络代理状态：未配置。无代理状态下很可能无法访问OpenAI家族的模型。建议：检查USE_PROXY选项是否修改。')
|
||||
log亮红('[PROXY] 网络代理状态：未配置。无代理状态下很可能无法访问OpenAI家族的模型。建议：检查USE_PROXY选项是否修改。')
|
||||
else:
|
||||
print亮绿('[PROXY] 网络代理状态：已配置。配置信息如下：', r)
|
||||
log亮绿('[PROXY] 网络代理状态：已配置。配置信息如下：', str(r))
|
||||
assert isinstance(r, dict), 'proxies格式错误，请注意proxies选项的格式，不要遗漏括号。'
|
||||
return r
|
||||
|
||||
|
||||
@@ -90,23 +90,6 @@ def make_history_cache():
|
||||
|
||||
|
||||
|
||||
# """
|
||||
# with gr.Row():
|
||||
# txt = gr.Textbox(show_label=False, placeholder="Input question here.", elem_id='user_input_main').style(container=False)
|
||||
# txtx = gr.Textbox(show_label=False, placeholder="Input question here.", elem_id='user_input_main').style(container=False)
|
||||
# with gr.Row():
|
||||
# btn_value = "Test"
|
||||
# elem_id = "TestCase"
|
||||
# variant = "primary"
|
||||
# input_list = [txt, txtx]
|
||||
# output_list = [txt, txtx]
|
||||
# input_name_list = ["txt(input)", "txtx(input)"]
|
||||
# output_name_list = ["txt", "txtx"]
|
||||
# js_callback = """(txt, txtx)=>{console.log(txt); console.log(txtx);}"""
|
||||
# def function(txt, txtx):
|
||||
# return "booo", "goooo"
|
||||
# create_button_with_javascript_callback(btn_value, elem_id, variant, js_callback, input_list, output_list, function, input_name_list, output_name_list)
|
||||
# """
|
||||
def create_button_with_javascript_callback(btn_value, elem_id, variant, js_callback, input_list, output_list, function, input_name_list, output_name_list):
|
||||
import gradio as gr
|
||||
middle_ware_component = gr.Textbox(visible=False, elem_id=elem_id+'_buffer')
|
||||
|
||||
@@ -8,6 +8,7 @@ import gradio
|
||||
import shutil
|
||||
import glob
|
||||
from shared_utils.config_loader import get_conf
|
||||
from loguru import logger
|
||||
|
||||
def html_local_file(file):
|
||||
base_path = os.path.dirname(__file__) # 项目目录
|
||||
@@ -100,7 +101,7 @@ def extract_archive(file_path, dest_dir):
|
||||
with zipfile.ZipFile(file_path, "r") as zipobj:
|
||||
zipobj._extract_member = lambda a,b,c: zip_extract_member_new(zipobj, a,b,c) # 修复中文乱码的问题
|
||||
zipobj.extractall(path=dest_dir)
|
||||
print("Successfully extracted zip archive to {}".format(dest_dir))
|
||||
logger.info("Successfully extracted zip archive to {}".format(dest_dir))
|
||||
|
||||
elif file_extension in [".tar", ".gz", ".bz2"]:
|
||||
with tarfile.open(file_path, "r:*") as tarobj:
|
||||
@@ -113,7 +114,7 @@ def extract_archive(file_path, dest_dir):
|
||||
raise Exception(f"Attempted Path Traversal in {member.name}")
|
||||
|
||||
tarobj.extractall(path=dest_dir)
|
||||
print("Successfully extracted tar archive to {}".format(dest_dir))
|
||||
logger.info("Successfully extracted tar archive to {}".format(dest_dir))
|
||||
|
||||
# 第三方库，需要预先pip install rarfile
|
||||
# 此外，Windows上还需要安装winrar软件，配置其Path环境变量，如"C:\Program Files\WinRAR"才可以
|
||||
@@ -123,9 +124,9 @@ def extract_archive(file_path, dest_dir):
|
||||
|
||||
with rarfile.RarFile(file_path) as rf:
|
||||
rf.extractall(path=dest_dir)
|
||||
print("Successfully extracted rar archive to {}".format(dest_dir))
|
||||
logger.info("Successfully extracted rar archive to {}".format(dest_dir))
|
||||
except:
|
||||
print("Rar format requires additional dependencies to install")
|
||||
logger.info("Rar format requires additional dependencies to install")
|
||||
return "\n\n解压失败! 需要安装pip install rarfile来解压rar文件。建议：使用zip压缩格式。"
|
||||
|
||||
# 第三方库，需要预先pip install py7zr
|
||||
@@ -135,9 +136,9 @@ def extract_archive(file_path, dest_dir):
|
||||
|
||||
with py7zr.SevenZipFile(file_path, mode="r") as f:
|
||||
f.extractall(path=dest_dir)
|
||||
print("Successfully extracted 7z archive to {}".format(dest_dir))
|
||||
logger.info("Successfully extracted 7z archive to {}".format(dest_dir))
|
||||
except:
|
||||
print("7z format requires additional dependencies to install")
|
||||
logger.info("7z format requires additional dependencies to install")
|
||||
return "\n\n解压失败! 需要安装pip install py7zr来解压7z文件"
|
||||
else:
|
||||
return ""
|
||||
|
||||
@@ -34,6 +34,9 @@ def is_api2d_key(key):
|
||||
API_MATCH_API2D = re.match(r"fk[a-zA-Z0-9]{6}-[a-zA-Z0-9]{32}$", key)
|
||||
return bool(API_MATCH_API2D)
|
||||
|
||||
def is_openroute_api_key(key):
|
||||
API_MATCH_OPENROUTE = re.match(r"sk-or-v1-[a-zA-Z0-9]{64}$", key)
|
||||
return bool(API_MATCH_OPENROUTE)
|
||||
|
||||
def is_cohere_api_key(key):
|
||||
API_MATCH_AZURE = re.match(r"[a-zA-Z0-9]{40}$", key)
|
||||
@@ -89,6 +92,10 @@ def select_api_key(keys, llm_model):
|
||||
if llm_model.startswith('cohere-'):
|
||||
for k in key_list:
|
||||
if is_cohere_api_key(k): avail_key_list.append(k)
|
||||
|
||||
if llm_model.startswith('openrouter-'):
|
||||
for k in key_list:
|
||||
if is_openroute_api_key(k): avail_key_list.append(k)
|
||||
|
||||
if len(avail_key_list) == 0:
|
||||
raise RuntimeError(f"您提供的api-key不满足要求，不包含任何可用于{llm_model}的api-key。您可能选择了错误的模型或请求源（左上角更换模型菜单中可切换openai,azure,claude,cohere等请求源）。")
|
||||
|
||||
69
shared_utils/logging.py
Normal file
69
shared_utils/logging.py
Normal file
@@ -0,0 +1,69 @@
|
||||
from loguru import logger
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
|
||||
def chat_log_filter(record):
|
||||
return "chat_msg" in record["extra"]
|
||||
|
||||
def not_chat_log_filter(record):
|
||||
return "chat_msg" not in record["extra"]
|
||||
|
||||
def formatter_with_clip(record):
|
||||
# Note this function returns the string to be formatted, not the actual message to be logged
|
||||
# record["extra"]["serialized"] = "555555"
|
||||
max_len = 12
|
||||
record['function_x'] = record['function'].center(max_len)
|
||||
if len(record['function_x']) > max_len:
|
||||
record['function_x'] = ".." + record['function_x'][-(max_len-2):]
|
||||
record['line_x'] = str(record['line']).ljust(3)
|
||||
return '<green>{time:HH:mm}</green> | <cyan>{function_x}</cyan>:<cyan>{line_x}</cyan> | <level>{message}</level>\n'
|
||||
|
||||
def setup_logging(PATH_LOGGING):
|
||||
|
||||
admin_log_path = os.path.join(PATH_LOGGING, "admin")
|
||||
os.makedirs(admin_log_path, exist_ok=True)
|
||||
sensitive_log_path = os.path.join(admin_log_path, "chat_secrets.log")
|
||||
regular_log_path = os.path.join(admin_log_path, "console_log.log")
|
||||
logger.remove()
|
||||
logger.configure(
|
||||
levels=[dict(name="WARNING", color="<g>")],
|
||||
)
|
||||
|
||||
logger.add(
|
||||
sys.stderr,
|
||||
format=formatter_with_clip,
|
||||
# format='<green>{time:HH:mm}</green> | <cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>',
|
||||
filter=(lambda record: not chat_log_filter(record)),
|
||||
colorize=True,
|
||||
enqueue=True
|
||||
)
|
||||
|
||||
logger.add(
|
||||
sensitive_log_path,
|
||||
format='<green>{time:MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>',
|
||||
rotation="10 MB",
|
||||
filter=chat_log_filter,
|
||||
enqueue=True,
|
||||
)
|
||||
|
||||
logger.add(
|
||||
regular_log_path,
|
||||
format='<green>{time:MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>',
|
||||
rotation="10 MB",
|
||||
filter=not_chat_log_filter,
|
||||
enqueue=True,
|
||||
)
|
||||
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
|
||||
logger.warning(f"所有对话记录将自动保存在本地目录{sensitive_log_path}, 请注意自我隐私保护哦！")
|
||||
|
||||
|
||||
# logger.bind(chat_msg=True).info("This message is logged to the file!")
|
||||
# logger.debug(f"debug message")
|
||||
# logger.info(f"info message")
|
||||
# logger.success(f"success message")
|
||||
# logger.error(f"error message")
|
||||
# logger.add("special.log", filter=lambda record: "special" in record["extra"])
|
||||
# logger.debug("This message is not logged to the file")
|
||||
Reference in New Issue
Block a user