You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
14 lines
360 B
14 lines
360 B
from swarms.chunkers.base_chunker import BaseChunker
|
|
from swarms.chunkers.chunk_seperator import ChunkSeparator
|
|
|
|
|
|
class TextChunker(BaseChunker):
|
|
DEFAULT_SEPARATORS = [
|
|
ChunkSeparator("\n\n"),
|
|
ChunkSeparator("\n"),
|
|
ChunkSeparator(". "),
|
|
ChunkSeparator("! "),
|
|
ChunkSeparator("? "),
|
|
ChunkSeparator(" "),
|
|
]
|