set default model properly

pull/570/head
Richard Anthony Hein 8 months ago
parent 85087aecd8
commit 705ca9803c

@ -12,7 +12,7 @@ weather_api_key= "af6ef989b5c50a91ca068cc00df125b7", # Replace with your weathe
llm = OpenAIChatLLM( llm = OpenAIChatLLM(
base_url=api_base, base_url=api_base,
api_key=api_key, api_key=api_key,
model="NousResearch/Meta-Llama-3.1-8B-Instruct", model="NousResearch/Meta-Llama-3-8B-Instruct",
temperature=0, temperature=0,
streaming=False streaming=False
) )

@ -5,4 +5,4 @@ docker run --runtime nvidia --gpus all \
--network=host \ --network=host \
--name vllm \ --name vllm \
vllm/vllm-openai:latest \ vllm/vllm-openai:latest \
--model NousResearch/Meta-Llama-3.1-8B-Instruct --model NousResearch/Meta-Llama-3-8B-Instruct

@ -44,8 +44,8 @@ class ChatRequest(BaseModel):
""" The model for a ChatRequest for theChatbot Chat POST endpoint""" """ The model for a ChatRequest for theChatbot Chat POST endpoint"""
id: str id: str
model: AIModel = AIModel( model: AIModel = AIModel(
id="NousResearch/Meta-Llama-3.1-8B-Instruct", id="NousResearch/Meta-Llama-3-8B-Instruct",
name="NousResearch/Meta-Llama-3.1-8B-Instruct", name="NousResearch/Meta-Llama-3-8B-Instruct",
maxLength=2048, maxLength=2048,
tokenLimit=2048, tokenLimit=2048,
) )

@ -11,7 +11,7 @@ api_base = os.getenv("OPENAI_API_BASE") or "http://localhost:8000/v1" # for vllm
# Create an instance of the OpenAIChat class # Create an instance of the OpenAIChat class
model = OpenAIChat( model = OpenAIChat(
base_url=api_base, api_key=api_key, model="NousResearch/Meta-Llama-3.1-8B-Instruct", temperature=0.5, streaming=True, verbose=True base_url=api_base, api_key=api_key, model="NousResearch/Meta-Llama-3-8B-Instruct", temperature=0.5, streaming=True, verbose=True
) )
# Initialize the agent # Initialize the agent

Loading…
Cancel
Save