unify tiktoken model

2023-04-17 19:41:50 +08:00
parent 40bc865d33
commit 2472185de9
8 changed files with 9 additions and 9 deletions
--- a/crazy_functions/代码重写为全英文_多线程.py
+++ b/crazy_functions/代码重写为全英文_多线程.py
@@ -61,7 +61,7 @@ def 全项目切换英文(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_
    MAX_TOKEN = 3000
    import tiktoken
    from toolbox import get_conf
-    enc = tiktoken.encoding_for_model(*get_conf('LLM_MODEL'))
+    enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
    def get_token_fn(txt): return len(enc.encode(txt, disallowed_special=()))