|
|
|
@ -1155,17 +1155,6 @@ class OpenAIChat(BaseLLM):
|
|
|
|
|
|
|
|
|
|
def get_token_ids(self, text: str) -> List[int]:
|
|
|
|
|
"""Get the token IDs using the tiktoken package."""
|
|
|
|
|
# tiktoken NOT supported for Python < 3.8
|
|
|
|
|
if sys.version_info[1] < 8:
|
|
|
|
|
return super().get_token_ids(text)
|
|
|
|
|
try:
|
|
|
|
|
import tiktoken
|
|
|
|
|
except ImportError:
|
|
|
|
|
raise ImportError(
|
|
|
|
|
"Could not import tiktoken python package. This is"
|
|
|
|
|
" needed in order to calculate get_num_tokens. Please"
|
|
|
|
|
" install it with `pip install tiktoken`."
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
enc = tiktoken.encoding_for_model(self.model_name)
|
|
|
|
|
return enc.encode(
|
|
|
|
|