|
|
@ -16,5 +16,13 @@ def count_tokens(text: str, model: str = "gpt-4o") -> int:
|
|
|
|
return len(encode(model=model, text=text))
|
|
|
|
return len(encode(model=model, text=text))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class LiteLLMTokenizer:
|
|
|
|
|
|
|
|
def __init__(self, model_name="gpt-4o"):
|
|
|
|
|
|
|
|
self.model_name = model_name
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def count(self, text):
|
|
|
|
|
|
|
|
return count_tokens(text, model=self.model_name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# if __name__ == "__main__":
|
|
|
|
# if __name__ == "__main__":
|
|
|
|
# print(count_tokens("Hello, how are you?"))
|
|
|
|
# print(count_tokens("Hello, how are you?"))
|
|
|
|