no quantization

pull/53/head
Kye 1 year ago
parent 4469164e1e
commit 692a4557b1

@ -4,7 +4,6 @@ from swarms.orchestrator.autoscaler import AutoScaler
# worker # worker
# from swarms.workers.worker_node import WorkerNode # from swarms.workers.worker_node import WorkerNode
from swarms.workers.worker import Worker from swarms.workers.worker import Worker
from swarms.workers.autobot import AutoBot
#boss #boss
from swarms.boss.boss_node import BossNode from swarms.boss.boss_node import BossNode

@ -5,7 +5,13 @@ import logging
from transformers import AutoModelForCausalLM, AutoTokenizer from transformers import AutoModelForCausalLM, AutoTokenizer
class HuggingFaceLLM: class HuggingFaceLLM:
def __init__(self, model_id: str, device: str = None, max_length: int = 20, quantize: bool = False, quantization_config: dict = None): def __init__(self,
model_id: str,
device: str = None,
max_length: int = 20,
quantize: bool = False,
quantization_config: dict = None
):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.device = device if device else ('cuda' if torch.cuda.is_available() else 'cpu') self.device = device if device else ('cuda' if torch.cuda.is_available() else 'cpu')
self.model_id = model_id self.model_id = model_id

Loading…
Cancel
Save