CC/gpt_academic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

improve long text breakdown perfomance

Browse Source
This commit is contained in:
binary-husky
2023-12-19 19:30:44 +08:00
parent 6e9936531d
commit a0bfa7ba1c
13 changed files with 186 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crazy_functions/批量Markdown翻译.py
View File

@@ -28,8 +28,8 @@ class PaperFileGroup():
self.sp_file_index.append(index)
self.sp_file_tag.append(self.file_paths[index])
else:
from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit)
from crazy_functions.pdf_fns.breakdown_txt import breakdown_text_to_satisfy_token_limit
segments = breakdown_text_to_satisfy_token_limit(file_content, max_token_limit)
for j, segment in enumerate(segments):
self.sp_file_contents.append(segment)
self.sp_file_index.append(index)