CC/gpt_academic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

move token limit conf to bridge_all.py

Browse Source
This commit is contained in:
binary-husky
2023-12-04 10:39:10 +08:00
parent 9bfc3400f9
commit 3c03f240ba
3 changed files with 4 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
request_llms/bridge_all.py
View File

@@ -552,7 +552,7 @@ if "deepseekcoder" in AVAIL_LLM_MODELS: # deepseekcoder
"fn_with_ui": deepseekcoder_ui,
"fn_without_ui": deepseekcoder_noui,
"endpoint": None,
"max_token": 4096,
"max_token": 2048,
"tokenizer": tokenizer_gpt35,
"token_cnt": get_token_num_gpt35,
}

5
request_llms/bridge_deepseekcoder.py
View File

@@ -8,7 +8,6 @@ from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns
from threading import Thread
import torch
MAX_INPUT_TOKEN_LENGTH = get_conf("MAX_INPUT_TOKEN_LENGTH")
def download_huggingface_model(model_name, max_retry, local_dir):
from huggingface_hub import snapshot_download
for i in range(1, max_retry):
@@ -94,8 +93,8 @@ class GetCoderLMHandle(LocalLLMHandle):
history.append({ 'role': 'user', 'content': query})
messages = history
inputs = self._tokenizer.apply_chat_template(messages, return_tensors="pt")
if inputs.shape[1] > MAX_INPUT_TOKEN_LENGTH:
inputs = inputs[:, -MAX_INPUT_TOKEN_LENGTH:]
if inputs.shape[1] > max_length:
inputs = inputs[:, -max_length:]
inputs = inputs.to(self._model.device)
generation_kwargs = dict(
inputs=inputs,