CC/gpt_academic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

tiktoken做lazyload处理

Browse Source
This commit is contained in:
Your Name
2023-04-19 14:27:34 +08:00
parent 28aa6d1dc0
commit b0409b929b
12 changed files with 83 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crazy_functions/理解PDF文档内容.py
View File

@@ -17,8 +17,8 @@ def 解析PDF(file_name, llm_kwargs, plugin_kwargs, chatbot, history, system_pro
TOKEN_LIMIT_PER_FRAGMENT = 2500
from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
from toolbox import get_conf
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
from request_llm.bridge_all import model_info
enc = model_info["gpt-3.5-turbo"]['tokenizer']
def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf(
txt=file_content, get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT)