unify tiktoken model

2023-04-17 19:41:50 +08:00
parent 40bc865d33
commit 2472185de9
8 changed files with 9 additions and 9 deletions
--- a/crazy_functions/Latex全文翻译.py
+++ b/crazy_functions/Latex全文翻译.py
@@ -13,7 +13,7 @@ class PaperFileGroup():
        # count_token
        import tiktoken
        from toolbox import get_conf
-        enc = tiktoken.encoding_for_model(*get_conf('LLM_MODEL'))
+        enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
        def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
        self.get_token_num = get_token_num