change some open fn encoding to utf-8
This commit is contained in:
@@ -26,7 +26,7 @@ class GetLlamaHandle(LocalLLMHandle):
|
||||
import platform
|
||||
huggingface_token, device = get_conf('HUGGINGFACE_ACCESS_TOKEN', 'LOCAL_MODEL_DEVICE')
|
||||
assert len(huggingface_token) != 0, "没有填写 HUGGINGFACE_ACCESS_TOKEN"
|
||||
with open(os.path.expanduser('~/.cache/huggingface/token'), 'w') as f:
|
||||
with open(os.path.expanduser('~/.cache/huggingface/token'), 'w', encoding='utf8') as f:
|
||||
f.write(huggingface_token)
|
||||
model_id = 'meta-llama/Llama-2-7b-chat-hf'
|
||||
with ProxyNetworkActivate('Download_LLM'):
|
||||
|
||||
@@ -31,7 +31,7 @@ class MoonShotInit:
|
||||
files.append(f)
|
||||
for file in files:
|
||||
if file.split('.')[-1] in ['pdf']:
|
||||
with open(file, 'r') as fp:
|
||||
with open(file, 'r', encoding='utf8') as fp:
|
||||
from crazy_functions.crazy_utils import read_and_clean_pdf_text
|
||||
file_content, _ = read_and_clean_pdf_text(fp)
|
||||
what_ask.append({"role": "system", "content": file_content})
|
||||
|
||||
Reference in New Issue
Block a user