disallow special token + limit num of file < 512

2023-04-14 09:50:14 +08:00
parent a2002ebd85
commit dd648bd446
8 changed files with 10 additions and 9 deletions
--- a/crazy_functions/Latex全文翻译.py
+++ b/crazy_functions/Latex全文翻译.py
@@ -14,7 +14,7 @@ class PaperFileGroup():
        import tiktoken
        from toolbox import get_conf
        enc = tiktoken.encoding_for_model(*get_conf('LLM_MODEL'))
-        def get_token_num(txt): return len(enc.encode(txt))
+        def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
        self.get_token_num = get_token_num

    def run_file_split(self, max_token_limit=1900):