feat: fix model loading

app.py

@@ -127,8 +127,7 @@ chat_history = ""
 
 MAX_SLEEP_TIME = 40
 
-def download_model(model_url: str, memory_utilization: int):
-    # Extract model name from the URL
+def download_model(model_url: str, memory_utilization: int , model_dir: str):
     model_name = model_url.split('/')[-1]
     # Download the model using VLLM
     vllm_model = VLLM(

swarms/tools/singletool.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from langchain.llms import OpenAI
 from langchain import OpenAI, LLMChain, PromptTemplate, SerpAPIWrapper
 from langchain.agents import ZeroShotAgent, AgentExecutor, initialize_agent, Tool
@@ -5,11 +6,13 @@ import importlib
 import json
 import os
 import requests
+from vllm import LLM
 import yaml
 from .apitool import RequestTool
 from .executor import Executor, AgentExecutorWithTranslation
 from swarms.utils import get_logger
 from .BabyagiTools import BabyAGI
+from langchain.llms import VLLM
 # from .models.customllm import CustomLLM
 
 
@@ -62,6 +65,9 @@ def load_single_tools(tool_name, tool_url):
 
     return tool_name, tool_config_json
 
+# Read the model/ directory and get the list of models
+model_dir = Path("./models/")
+available_models = ["ChatGPT", "GPT-3.5"] + [f.name for f in model_dir.iterdir() if f.is_dir()]
 
 class STQuestionAnswerer:
     def __init__(self, openai_api_key="", stream_output=False, llm="ChatGPT"):
@@ -83,6 +89,8 @@ class STQuestionAnswerer:
             self.llm = OpenAI(
                 model_name="gpt-3.5-turbo", temperature=0.0, openai_api_key=key
             )  # use chatgpt
+        elif self.llm_model in available_models:
+            self.llm = VLLM(model=f"models/{self.llm_model}")
         else:
             raise RuntimeError("Your model is not available.")
 

swarms/tools/tools_controller.py

@@ -12,6 +12,7 @@ from .executor import Executor, AgentExecutorWithTranslation
 from vllm import LLM 
 from swarms.utils import get_logger
 from pathlib import Path
+from langchain.llms import VLLM
 
 logger = get_logger(__name__)
 
@@ -45,7 +46,7 @@ class MTQuestionAnswerer:
         self.openai_api_key = openai_api_key
         self.stream_output = stream_output
         self.llm_model = llm
-        self.model_path = model_path
+        self.model = model_path
         self.set_openai_api_key(openai_api_key)
         self.load_tools(all_tools)
 
@@ -59,7 +60,7 @@ class MTQuestionAnswerer:
                 model_name="gpt-3.5-turbo", temperature=0.0, openai_api_key=key
             )  # use chatgpt
         elif self.llm_model in available_models:  # If the selected model is a vLLM model
-            self.llm = LLM(model_path=f"model/{self.llm_model}")  # Load the vLLM model
+            self.llm = VLLM(model=f"models/{self.llm_model}")
         else:
             raise RuntimeError("Your model is not available.")