swarms/tests/tokenizers/test_anthropictokenizer.py

# AnthropicTokenizer

import pytest

from swarms.tokenizers.anthropic_tokenizer import AnthropicTokenizer


def test_post_init():
    tokenizer = AnthropicTokenizer()
    assert tokenizer.model == "claude-2.1"
    assert tokenizer.max_tokens == 200000


def test_default_max_tokens():
    tokenizer = AnthropicTokenizer(model="claude")
    assert tokenizer.default_max_tokens() == 100000


@pytest.mark.parametrize(
    "model,tokens", [("claude-2.1", 200000), ("claude", 100000)]
)
def test_default_max_tokens_models(model, tokens):
    tokenizer = AnthropicTokenizer(model=model)
    assert tokenizer.default_max_tokens() == tokens


def test_count_tokens_string():
    # Insert mock instantiation of anthropic client and its count_tokens function
    text = "This is a test string."
    tokenizer = AnthropicTokenizer()
    tokens = tokenizer.count_tokens(text)
    assert tokens == 5


def test_count_tokens_list():
    # Insert mock instantiation of anthropic client and its count_tokens function
    text = ["This", "is", "a", "test", "string."]
    tokenizer = AnthropicTokenizer()
    with pytest.raises(ValueError):
        tokenizer.count_tokens(text)