diff --git a/pyproject.toml b/pyproject.toml index 31bc1b00..6b4d96f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ [tool.poetry.dependencies] -python = "^3.8" +python = "^3.8.1" torch = "2.1.1" transformers = "4.37.1" openai = "0.28.0" diff --git a/swarms/models/openai_models.py b/swarms/models/openai_models.py index b1aa0117..a0ad35a3 100644 --- a/swarms/models/openai_models.py +++ b/swarms/models/openai_models.py @@ -1155,17 +1155,6 @@ class OpenAIChat(BaseLLM): def get_token_ids(self, text: str) -> List[int]: """Get the token IDs using the tiktoken package.""" - # tiktoken NOT supported for Python < 3.8 - if sys.version_info[1] < 8: - return super().get_token_ids(text) - try: - import tiktoken - except ImportError: - raise ImportError( - "Could not import tiktoken python package. This is" - " needed in order to calculate get_num_tokens. Please" - " install it with `pip install tiktoken`." - ) enc = tiktoken.encoding_for_model(self.model_name) return enc.encode(