change some open fn encoding to utf-8

This commit is contained in:
binary-husky
2024-11-19 15:49:29 +00:00
parent 588b22e039
commit e62decac21
4 changed files with 5 additions and 5 deletions

View File

@@ -26,7 +26,7 @@ class GetLlamaHandle(LocalLLMHandle):
import platform
huggingface_token, device = get_conf('HUGGINGFACE_ACCESS_TOKEN', 'LOCAL_MODEL_DEVICE')
assert len(huggingface_token) != 0, "没有填写 HUGGINGFACE_ACCESS_TOKEN"
with open(os.path.expanduser('~/.cache/huggingface/token'), 'w') as f:
with open(os.path.expanduser('~/.cache/huggingface/token'), 'w', encoding='utf8') as f:
f.write(huggingface_token)
model_id = 'meta-llama/Llama-2-7b-chat-hf'
with ProxyNetworkActivate('Download_LLM'):

View File

@@ -31,7 +31,7 @@ class MoonShotInit:
files.append(f)
for file in files:
if file.split('.')[-1] in ['pdf']:
with open(file, 'r') as fp:
with open(file, 'r', encoding='utf8') as fp:
from crazy_functions.crazy_utils import read_and_clean_pdf_text
file_content, _ = read_and_clean_pdf_text(fp)
what_ask.append({"role": "system", "content": file_content})