unify tiktoken model

2023-04-17 19:41:50 +08:00
parent 40bc865d33
commit 2472185de9
8 changed files with 9 additions and 9 deletions
--- a/crazy_functions/理解PDF文档内容.py
+++ b/crazy_functions/理解PDF文档内容.py
@@ -18,7 +18,7 @@ def 解析PDF(file_name, llm_kwargs, plugin_kwargs, chatbot, history, system_pro

    from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
    from toolbox import get_conf
-    enc = tiktoken.encoding_for_model(*get_conf('LLM_MODEL'))
+    enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
    def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
    paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf(
        txt=file_content,  get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT)