Merge branch 'master' into frontier
This commit is contained in:
binary-husky
@@ -179,6 +179,24 @@ model_info = {
|
||||
"token_cnt": get_token_num_gpt4,
|
||||
},
|
||||
|
||||
"gpt-4o": {
|
||||
"fn_with_ui": chatgpt_ui,
|
||||
"fn_without_ui": chatgpt_noui,
|
||||
"endpoint": openai_endpoint,
|
||||
"max_token": 128000,
|
||||
"tokenizer": tokenizer_gpt4,
|
||||
"token_cnt": get_token_num_gpt4,
|
||||
},
|
||||
|
||||
"gpt-4o-2024-05-13": {
|
||||
"fn_with_ui": chatgpt_ui,
|
||||
"fn_without_ui": chatgpt_noui,
|
||||
"endpoint": openai_endpoint,
|
||||
"max_token": 128000,
|
||||
"tokenizer": tokenizer_gpt4,
|
||||
"token_cnt": get_token_num_gpt4,
|
||||
},
|
||||
|
||||
"gpt-4-turbo-preview": {
|
||||
"fn_with_ui": chatgpt_ui,
|
||||
"fn_without_ui": chatgpt_noui,
|
||||
@@ -971,6 +989,13 @@ if len(AZURE_CFG_ARRAY) > 0:
|
||||
AVAIL_LLM_MODELS += [azure_model_name]
|
||||
|
||||
|
||||
# -=-=-=-=-=-=--=-=-=-=-=-=--=-=-=-=-=-=--=-=-=-=-=-=-=-=
|
||||
# -=-=-=-=-=-=-=-=-=- ☝️ 以上是模型路由 -=-=-=-=-=-=-=-=-=
|
||||
# -=-=-=-=-=-=--=-=-=-=-=-=--=-=-=-=-=-=--=-=-=-=-=-=-=-=
|
||||
|
||||
# -=-=-=-=-=-=--=-=-=-=-=-=--=-=-=-=-=-=--=-=-=-=-=-=-=-=
|
||||
# -=-=-=-=-=-=-= 👇 以下是多模型路由切换函数 -=-=-=-=-=-=-=
|
||||
# -=-=-=-=-=-=--=-=-=-=-=-=--=-=-=-=-=-=--=-=-=-=-=-=-=-=
|
||||
|
||||
|
||||
def LLM_CATCH_EXCEPTION(f):
|
||||
@@ -1007,13 +1032,11 @@ def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list, sys
|
||||
model = llm_kwargs['llm_model']
|
||||
n_model = 1
|
||||
if '&' not in model:
|
||||
|
||||
# 如果只询问1个大语言模型：
|
||||
# 如果只询问“一个”大语言模型（多数情况）：
|
||||
method = model_info[model]["fn_without_ui"]
|
||||
return method(inputs, llm_kwargs, history, sys_prompt, observe_window, console_slience)
|
||||
else:
|
||||
|
||||
# 如果同时询问多个大语言模型，这个稍微啰嗦一点，但思路相同，您不必读这个else分支
|
||||
# 如果同时询问“多个”大语言模型，这个稍微啰嗦一点，但思路相同，您不必读这个else分支
|
||||
executor = ThreadPoolExecutor(max_workers=4)
|
||||
models = model.split('&')
|
||||
n_model = len(models)
|
||||
@@ -1066,8 +1089,26 @@ def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list, sys
|
||||
res = '<br/><br/>\n\n---\n\n'.join(return_string_collect)
|
||||
return res
|
||||
|
||||
# 根据基础功能区 ModelOverride 参数调整模型类型，用于 `predict` 中
|
||||
import importlib
|
||||
import core_functional
|
||||
def execute_model_override(llm_kwargs, additional_fn, method):
|
||||
functional = core_functional.get_core_functions()
|
||||
if (additional_fn in functional) and 'ModelOverride' in functional[additional_fn]:
|
||||
# 热更新Prompt & ModelOverride
|
||||
importlib.reload(core_functional)
|
||||
functional = core_functional.get_core_functions()
|
||||
model_override = functional[additional_fn]['ModelOverride']
|
||||
if model_override not in model_info:
|
||||
raise ValueError(f"模型覆盖参数 '{model_override}' 指向一个暂不支持的模型，请检查配置文件。")
|
||||
method = model_info[model_override]["fn_with_ui"]
|
||||
llm_kwargs['llm_model'] = model_override
|
||||
return llm_kwargs, additional_fn, method
|
||||
# 默认返回原参数
|
||||
return llm_kwargs, additional_fn, method
|
||||
|
||||
def predict(inputs:str, llm_kwargs:dict, *args, **kwargs):
|
||||
def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot,
|
||||
history:list=[], system_prompt:str='', stream:bool=True, additional_fn:str=None):
|
||||
"""
|
||||
发送至LLM，流式获取输出。
|
||||
用于基础的对话功能。
|
||||
@@ -1086,6 +1127,11 @@ def predict(inputs:str, llm_kwargs:dict, *args, **kwargs):
|
||||
"""
|
||||
|
||||
inputs = apply_gpt_academic_string_mask(inputs, mode="show_llm")
|
||||
method = model_info[llm_kwargs['llm_model']]["fn_with_ui"] # 如果这里报错，检查config中的AVAIL_LLM_MODELS选项
|
||||
yield from method(inputs, llm_kwargs, *args, **kwargs)
|
||||
|
||||
method = model_info[llm_kwargs['llm_model']]["fn_with_ui"] # 如果这里报错，检查config中的AVAIL_LLM_MODELS选项
|
||||
|
||||
if additional_fn: # 根据基础功能区 ModelOverride 参数调整模型类型
|
||||
llm_kwargs, additional_fn, method = execute_model_override(llm_kwargs, additional_fn, method)
|
||||
|
||||
yield from method(inputs, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, stream, additional_fn)
|
||||
|
||||
|
||||
@@ -1,69 +1,69 @@
|
||||
import time
|
||||
from toolbox import update_ui, get_conf, update_ui_lastest_msg
|
||||
from toolbox import check_packages, report_exception
|
||||
|
||||
model_name = '云雀大模型'
|
||||
|
||||
def validate_key():
|
||||
YUNQUE_SECRET_KEY = get_conf("YUNQUE_SECRET_KEY")
|
||||
if YUNQUE_SECRET_KEY == '': return False
|
||||
return True
|
||||
|
||||
def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list=[], sys_prompt:str="",
|
||||
observe_window:list=[], console_slience:bool=False):
|
||||
"""
|
||||
⭐ 多线程方法
|
||||
函数的说明请见 request_llms/bridge_all.py
|
||||
"""
|
||||
watch_dog_patience = 5
|
||||
response = ""
|
||||
|
||||
if validate_key() is False:
|
||||
raise RuntimeError('请配置YUNQUE_SECRET_KEY')
|
||||
|
||||
from .com_skylark2api import YUNQUERequestInstance
|
||||
sri = YUNQUERequestInstance()
|
||||
for response in sri.generate(inputs, llm_kwargs, history, sys_prompt):
|
||||
if len(observe_window) >= 1:
|
||||
observe_window[0] = response
|
||||
if len(observe_window) >= 2:
|
||||
if (time.time()-observe_window[1]) > watch_dog_patience: raise RuntimeError("程序终止。")
|
||||
return response
|
||||
|
||||
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
|
||||
"""
|
||||
⭐ 单线程方法
|
||||
函数的说明请见 request_llms/bridge_all.py
|
||||
"""
|
||||
chatbot.append((inputs, ""))
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
|
||||
# 尝试导入依赖，如果缺少依赖，则给出安装建议
|
||||
try:
|
||||
check_packages(["zhipuai"])
|
||||
except:
|
||||
yield from update_ui_lastest_msg(f"导入软件依赖失败。使用该模型需要额外依赖，安装方法```pip install --upgrade zhipuai```。",
|
||||
chatbot=chatbot, history=history, delay=0)
|
||||
return
|
||||
|
||||
if validate_key() is False:
|
||||
yield from update_ui_lastest_msg(lastmsg="[Local Message] 请配置HUOSHAN_API_KEY", chatbot=chatbot, history=history, delay=0)
|
||||
return
|
||||
|
||||
if additional_fn is not None:
|
||||
from core_functional import handle_core_functionality
|
||||
inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
|
||||
|
||||
# 开始接收回复
|
||||
from .com_skylark2api import YUNQUERequestInstance
|
||||
sri = YUNQUERequestInstance()
|
||||
response = f"[Local Message] 等待{model_name}响应中 ..."
|
||||
for response in sri.generate(inputs, llm_kwargs, history, system_prompt):
|
||||
chatbot[-1] = (inputs, response)
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
|
||||
# 总结输出
|
||||
if response == f"[Local Message] 等待{model_name}响应中 ...":
|
||||
response = f"[Local Message] {model_name}响应异常 ..."
|
||||
history.extend([inputs, response])
|
||||
import time
|
||||
from toolbox import update_ui, get_conf, update_ui_lastest_msg
|
||||
from toolbox import check_packages, report_exception
|
||||
|
||||
model_name = '云雀大模型'
|
||||
|
||||
def validate_key():
|
||||
YUNQUE_SECRET_KEY = get_conf("YUNQUE_SECRET_KEY")
|
||||
if YUNQUE_SECRET_KEY == '': return False
|
||||
return True
|
||||
|
||||
def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list=[], sys_prompt:str="",
|
||||
observe_window:list=[], console_slience:bool=False):
|
||||
"""
|
||||
⭐ 多线程方法
|
||||
函数的说明请见 request_llms/bridge_all.py
|
||||
"""
|
||||
watch_dog_patience = 5
|
||||
response = ""
|
||||
|
||||
if validate_key() is False:
|
||||
raise RuntimeError('请配置YUNQUE_SECRET_KEY')
|
||||
|
||||
from .com_skylark2api import YUNQUERequestInstance
|
||||
sri = YUNQUERequestInstance()
|
||||
for response in sri.generate(inputs, llm_kwargs, history, sys_prompt):
|
||||
if len(observe_window) >= 1:
|
||||
observe_window[0] = response
|
||||
if len(observe_window) >= 2:
|
||||
if (time.time()-observe_window[1]) > watch_dog_patience: raise RuntimeError("程序终止。")
|
||||
return response
|
||||
|
||||
def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
|
||||
"""
|
||||
⭐ 单线程方法
|
||||
函数的说明请见 request_llms/bridge_all.py
|
||||
"""
|
||||
chatbot.append((inputs, ""))
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
|
||||
# 尝试导入依赖，如果缺少依赖，则给出安装建议
|
||||
try:
|
||||
check_packages(["zhipuai"])
|
||||
except:
|
||||
yield from update_ui_lastest_msg(f"导入软件依赖失败。使用该模型需要额外依赖，安装方法```pip install --upgrade zhipuai```。",
|
||||
chatbot=chatbot, history=history, delay=0)
|
||||
return
|
||||
|
||||
if validate_key() is False:
|
||||
yield from update_ui_lastest_msg(lastmsg="[Local Message] 请配置HUOSHAN_API_KEY", chatbot=chatbot, history=history, delay=0)
|
||||
return
|
||||
|
||||
if additional_fn is not None:
|
||||
from core_functional import handle_core_functionality
|
||||
inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot)
|
||||
|
||||
# 开始接收回复
|
||||
from .com_skylark2api import YUNQUERequestInstance
|
||||
sri = YUNQUERequestInstance()
|
||||
response = f"[Local Message] 等待{model_name}响应中 ..."
|
||||
for response in sri.generate(inputs, llm_kwargs, history, system_prompt):
|
||||
chatbot[-1] = (inputs, response)
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
|
||||
# 总结输出
|
||||
if response == f"[Local Message] 等待{model_name}响应中 ...":
|
||||
response = f"[Local Message] {model_name}响应异常 ..."
|
||||
history.extend([inputs, response])
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
@@ -1,95 +1,95 @@
|
||||
from toolbox import get_conf
|
||||
import threading
|
||||
import logging
|
||||
import os
|
||||
|
||||
timeout_bot_msg = '[Local Message] Request timeout. Network error.'
|
||||
#os.environ['VOLC_ACCESSKEY'] = ''
|
||||
#os.environ['VOLC_SECRETKEY'] = ''
|
||||
|
||||
class YUNQUERequestInstance():
|
||||
def __init__(self):
|
||||
|
||||
self.time_to_yield_event = threading.Event()
|
||||
self.time_to_exit_event = threading.Event()
|
||||
|
||||
self.result_buf = ""
|
||||
|
||||
def generate(self, inputs, llm_kwargs, history, system_prompt):
|
||||
# import _thread as thread
|
||||
from volcengine.maas import MaasService, MaasException
|
||||
|
||||
maas = MaasService('maas-api.ml-platform-cn-beijing.volces.com', 'cn-beijing')
|
||||
|
||||
YUNQUE_SECRET_KEY, YUNQUE_ACCESS_KEY,YUNQUE_MODEL = get_conf("YUNQUE_SECRET_KEY", "YUNQUE_ACCESS_KEY","YUNQUE_MODEL")
|
||||
maas.set_ak(YUNQUE_ACCESS_KEY) #填写 VOLC_ACCESSKEY
|
||||
maas.set_sk(YUNQUE_SECRET_KEY) #填写 'VOLC_SECRETKEY'
|
||||
|
||||
self.result_buf = ""
|
||||
|
||||
req = {
|
||||
"model": {
|
||||
"name": YUNQUE_MODEL,
|
||||
"version": "1.0", # use default version if not specified.
|
||||
},
|
||||
"parameters": {
|
||||
"max_new_tokens": 4000, # 输出文本的最大tokens限制
|
||||
"min_new_tokens": 1, # 输出文本的最小tokens限制
|
||||
"temperature": llm_kwargs['temperature'], # 用于控制生成文本的随机性和创造性，Temperature值越大随机性越大，取值范围0~1
|
||||
"top_p": llm_kwargs['top_p'], # 用于控制输出tokens的多样性，TopP值越大输出的tokens类型越丰富，取值范围0~1
|
||||
"top_k": 0, # 选择预测值最大的k个token进行采样，取值范围0-1000，0表示不生效
|
||||
"max_prompt_tokens": 4000, # 最大输入 token 数，如果给出的 prompt 的 token 长度超过此限制，取最后 max_prompt_tokens 个 token 输入模型。
|
||||
},
|
||||
"messages": self.generate_message_payload(inputs, llm_kwargs, history, system_prompt)
|
||||
}
|
||||
|
||||
response = maas.stream_chat(req)
|
||||
|
||||
for resp in response:
|
||||
self.result_buf += resp.choice.message.content
|
||||
yield self.result_buf
|
||||
'''
|
||||
for event in response.events():
|
||||
if event.event == "add":
|
||||
self.result_buf += event.data
|
||||
yield self.result_buf
|
||||
elif event.event == "error" or event.event == "interrupted":
|
||||
raise RuntimeError("Unknown error:" + event.data)
|
||||
elif event.event == "finish":
|
||||
yield self.result_buf
|
||||
break
|
||||
else:
|
||||
raise RuntimeError("Unknown error:" + str(event))
|
||||
|
||||
logging.info(f'[raw_input] {inputs}')
|
||||
logging.info(f'[response] {self.result_buf}')
|
||||
'''
|
||||
return self.result_buf
|
||||
|
||||
def generate_message_payload(inputs, llm_kwargs, history, system_prompt):
|
||||
from volcengine.maas import ChatRole
|
||||
conversation_cnt = len(history) // 2
|
||||
messages = [{"role": ChatRole.USER, "content": system_prompt},
|
||||
{"role": ChatRole.ASSISTANT, "content": "Certainly!"}]
|
||||
if conversation_cnt:
|
||||
for index in range(0, 2 * conversation_cnt, 2):
|
||||
what_i_have_asked = {}
|
||||
what_i_have_asked["role"] = ChatRole.USER
|
||||
what_i_have_asked["content"] = history[index]
|
||||
what_gpt_answer = {}
|
||||
what_gpt_answer["role"] = ChatRole.ASSISTANT
|
||||
what_gpt_answer["content"] = history[index + 1]
|
||||
if what_i_have_asked["content"] != "":
|
||||
if what_gpt_answer["content"] == "":
|
||||
continue
|
||||
if what_gpt_answer["content"] == timeout_bot_msg:
|
||||
continue
|
||||
messages.append(what_i_have_asked)
|
||||
messages.append(what_gpt_answer)
|
||||
else:
|
||||
messages[-1]['content'] = what_gpt_answer['content']
|
||||
what_i_ask_now = {}
|
||||
what_i_ask_now["role"] = ChatRole.USER
|
||||
what_i_ask_now["content"] = inputs
|
||||
messages.append(what_i_ask_now)
|
||||
from toolbox import get_conf
|
||||
import threading
|
||||
import logging
|
||||
import os
|
||||
|
||||
timeout_bot_msg = '[Local Message] Request timeout. Network error.'
|
||||
#os.environ['VOLC_ACCESSKEY'] = ''
|
||||
#os.environ['VOLC_SECRETKEY'] = ''
|
||||
|
||||
class YUNQUERequestInstance():
|
||||
def __init__(self):
|
||||
|
||||
self.time_to_yield_event = threading.Event()
|
||||
self.time_to_exit_event = threading.Event()
|
||||
|
||||
self.result_buf = ""
|
||||
|
||||
def generate(self, inputs, llm_kwargs, history, system_prompt):
|
||||
# import _thread as thread
|
||||
from volcengine.maas import MaasService, MaasException
|
||||
|
||||
maas = MaasService('maas-api.ml-platform-cn-beijing.volces.com', 'cn-beijing')
|
||||
|
||||
YUNQUE_SECRET_KEY, YUNQUE_ACCESS_KEY,YUNQUE_MODEL = get_conf("YUNQUE_SECRET_KEY", "YUNQUE_ACCESS_KEY","YUNQUE_MODEL")
|
||||
maas.set_ak(YUNQUE_ACCESS_KEY) #填写 VOLC_ACCESSKEY
|
||||
maas.set_sk(YUNQUE_SECRET_KEY) #填写 'VOLC_SECRETKEY'
|
||||
|
||||
self.result_buf = ""
|
||||
|
||||
req = {
|
||||
"model": {
|
||||
"name": YUNQUE_MODEL,
|
||||
"version": "1.0", # use default version if not specified.
|
||||
},
|
||||
"parameters": {
|
||||
"max_new_tokens": 4000, # 输出文本的最大tokens限制
|
||||
"min_new_tokens": 1, # 输出文本的最小tokens限制
|
||||
"temperature": llm_kwargs['temperature'], # 用于控制生成文本的随机性和创造性，Temperature值越大随机性越大，取值范围0~1
|
||||
"top_p": llm_kwargs['top_p'], # 用于控制输出tokens的多样性，TopP值越大输出的tokens类型越丰富，取值范围0~1
|
||||
"top_k": 0, # 选择预测值最大的k个token进行采样，取值范围0-1000，0表示不生效
|
||||
"max_prompt_tokens": 4000, # 最大输入 token 数，如果给出的 prompt 的 token 长度超过此限制，取最后 max_prompt_tokens 个 token 输入模型。
|
||||
},
|
||||
"messages": self.generate_message_payload(inputs, llm_kwargs, history, system_prompt)
|
||||
}
|
||||
|
||||
response = maas.stream_chat(req)
|
||||
|
||||
for resp in response:
|
||||
self.result_buf += resp.choice.message.content
|
||||
yield self.result_buf
|
||||
'''
|
||||
for event in response.events():
|
||||
if event.event == "add":
|
||||
self.result_buf += event.data
|
||||
yield self.result_buf
|
||||
elif event.event == "error" or event.event == "interrupted":
|
||||
raise RuntimeError("Unknown error:" + event.data)
|
||||
elif event.event == "finish":
|
||||
yield self.result_buf
|
||||
break
|
||||
else:
|
||||
raise RuntimeError("Unknown error:" + str(event))
|
||||
|
||||
logging.info(f'[raw_input] {inputs}')
|
||||
logging.info(f'[response] {self.result_buf}')
|
||||
'''
|
||||
return self.result_buf
|
||||
|
||||
def generate_message_payload(inputs, llm_kwargs, history, system_prompt):
|
||||
from volcengine.maas import ChatRole
|
||||
conversation_cnt = len(history) // 2
|
||||
messages = [{"role": ChatRole.USER, "content": system_prompt},
|
||||
{"role": ChatRole.ASSISTANT, "content": "Certainly!"}]
|
||||
if conversation_cnt:
|
||||
for index in range(0, 2 * conversation_cnt, 2):
|
||||
what_i_have_asked = {}
|
||||
what_i_have_asked["role"] = ChatRole.USER
|
||||
what_i_have_asked["content"] = history[index]
|
||||
what_gpt_answer = {}
|
||||
what_gpt_answer["role"] = ChatRole.ASSISTANT
|
||||
what_gpt_answer["content"] = history[index + 1]
|
||||
if what_i_have_asked["content"] != "":
|
||||
if what_gpt_answer["content"] == "":
|
||||
continue
|
||||
if what_gpt_answer["content"] == timeout_bot_msg:
|
||||
continue
|
||||
messages.append(what_i_have_asked)
|
||||
messages.append(what_gpt_answer)
|
||||
else:
|
||||
messages[-1]['content'] = what_gpt_answer['content']
|
||||
what_i_ask_now = {}
|
||||
what_i_ask_now["role"] = ChatRole.USER
|
||||
what_i_ask_now["content"] = inputs
|
||||
messages.append(what_i_ask_now)
|
||||
return messages
|
||||
Reference in New Issue
Block a user