From 4e757d5a7dc1b58fd4b63e7bc4beff7ccdb27e67 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 15 Apr 2024 19:17:04 -0400 Subject: [PATCH 1/2] [BUFG][Tool Usage][Agent] --- pyproject.toml | 2 +- swarms/models/open_router.py | 0 swarms/prompts/worker_prompt.py | 237 ++++++++++++++++++++++++++------ swarms/structs/agent.py | 39 +++--- tool.py | 57 ++++++++ 5 files changed, 272 insertions(+), 63 deletions(-) create mode 100644 swarms/models/open_router.py create mode 100644 tool.py diff --git a/pyproject.toml b/pyproject.toml index a19bbcc1..b9c2fc1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "swarms" -version = "4.7.9" +version = "4.8.1" description = "Swarms - Pytorch" license = "MIT" authors = ["Kye Gomez "] diff --git a/swarms/models/open_router.py b/swarms/models/open_router.py new file mode 100644 index 00000000..e69de29b diff --git a/swarms/prompts/worker_prompt.py b/swarms/prompts/worker_prompt.py index 410ecb2a..cbe43afc 100644 --- a/swarms/prompts/worker_prompt.py +++ b/swarms/prompts/worker_prompt.py @@ -1,5 +1,8 @@ import datetime from pydantic import BaseModel, Field +from swarms.tools.tool import BaseTool +from swarms.tools.tool_utils import scrape_tool_func_docs +from typing import List time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -23,54 +26,202 @@ class ResponseFormat(BaseModel): response_json = ResponseFormat.model_json_schema() -def worker_tools_sop_promp(name: str, memory: str, time=time): - out = f""" - You are {name}, - Your decisions must always be made independently without seeking user assistance. - Play to your strengths as an LLM and pursue simple strategies with no legal complications. - If you have completed all your tasks, make sure to use the 'finish' command. - - GOALS: - - 1. Hello, how are you? Create an image of how you are doing! - - Constraints: - - 1. ~4000 word limit for short term memory. Your short term memory is short, so immediately save important information to files. - 2. If you are unsure how you previously did something or want to recall past events, thinking about similar events will help you remember. - 3. No user assistance - 4. Exclusively use the commands listed in double quotes e.g. 'command name' - - Commands: - - 1. finish: use this to signal that you have finished all your objectives, args: 'response': 'final response to let people know you have finished your objectives' - - Resources: - - 1. Internet access for searches and information gathering. - 2. Long Term memory management. - 3. Agents for delegation of simple tasks. - 4. File output. +tool_usage_browser = """ + +```json +{ + "thoughts": { + "text": "To check the weather in Miami, I will use the browser tool to search for 'Miami weather'.", + "reasoning": "The browser tool allows me to search the web, so I can look up the current weather conditions in Miami.", + "plan": "Use the browser tool to search Google for 'Miami weather'. Parse the result to get the current temperature, conditions, etc. and format that into a readable weather report." + }, + "command": { + "name": "browser", + "args": { + "query": "Miami weather" + } + } +} +``` + +""" + +tool_usage_terminal = """ + +```json +{ + "thoughts": { + "text": "To check the weather in Miami, I will use the browser tool to search for 'Miami weather'.", + "reasoning": "The browser tool allows me to search the web, so I can look up the current weather conditions in Miami.", + "plan": "Use the browser tool to search Google for 'Miami weather'. Parse the result to get the current temperature, conditions, etc. and format that into a readable weather report." + }, + "command": { + "name": "terminal", + "args": { + "code": "uptime" + } + } +} +``` + +""" + + +browser_and_terminal_tool = """ +``` +{ + "thoughts": { + "text": "To analyze the latest stock market trends, I need to fetch current stock data and then process it using a script.", + "reasoning": "Using the browser tool to retrieve stock data ensures I have the most recent information. Following this, the terminal tool can run a script that analyzes this data to identify trends.", + "plan": "First, use the browser to get the latest stock prices. Then, use the terminal to execute a data analysis script on the fetched data." + }, + "commands": [ + { + "name": "browser", + "args": { + "query": "download latest stock data for NASDAQ" + } + }, + { + "name": "terminal", + "args": { + "cmd": "python analyze_stocks.py" + } + } + ] +} +``` + +""" + + +browser_and_terminal_tool_two = """ +``` +{ + "thoughts": { + "text": "To prepare a monthly budget report, I need current expenditure data, process it, and calculate the totals and averages.", + "reasoning": "The browser will fetch the latest expenditure data. The terminal will run a processing script to organize the data, and the calculator will be used to sum up expenses and compute averages.", + "plan": "Download the data using the browser, process it with a terminal command, and then calculate totals and averages using the calculator." + }, + "commands": [ + { + "name": "browser", + "args": { + "query": "download monthly expenditure data" + } + }, + { + "name": "terminal", + "args": { + "cmd": "python process_expenditures.py" + } + }, + { + "name": "calculator", + "args": { + "operation": "sum", + "numbers": "[output_from_process_expenditures]" + } + } + ] +} + +``` + +""" + + +# Function to parse tools and get their documentation +def parse_tools(tools: List[BaseTool] = []): + tool_docs = [] + for tool in tools: + tool_doc = scrape_tool_func_docs(tool) + tool_docs.append(tool_doc) + return tool_docs + + +# Function to generate the worker prompt +def tool_usage_worker_prompt( + current_time=time, tools: List[BaseTool] = [] +): + tool_docs = parse_tools(tools) + + prompt = f""" + **Date and Time**: {current_time} + + ### Constraints + - Only use the tools as specified in the instructions. + - Follow the command format strictly to avoid errors and ensure consistency. + - Only use the tools for the intended purpose as described in the SOP. + - Document your thoughts, reasoning, and plan before executing the command. + - Provide the output in JSON format within markdown code blocks. + - Review the output to ensure it matches the expected outcome. + - Only follow the instructions provided in the SOP and do not deviate from the specified tasks unless tool usage is not required. - Performance Evaluation: + ### Performance Evaluation + - **Efficiency**: Use tools to complete tasks with minimal steps. + - **Accuracy**: Ensure that commands are executed correctly to achieve the desired outcome. + - **Adaptability**: Be ready to adjust the use of tools based on task requirements and feedback. + + ### Tool Commands + 1. **Browser** + - **Purpose**: To retrieve information from the internet. + - **Usage**: + - `{{"name": "browser", "args": {{"query": "search query here"}}}}` + - Example: Fetch current weather in London. + - Command: `{{"name": "browser", "args": {{"query": "London weather"}}}}` + + 2. **Terminal** + - **Purpose**: To execute system commands. + - **Usage**: + - `{{"name": "terminal", "args": {{"cmd": "system command here"}}}}` + - Example: Check disk usage on a server. + - Command: `{{"name": "terminal", "args": {{"cmd": "df -h"}}}}` + + 3. **Custom Tool** (if applicable) + - **Purpose**: Describe specific functionality. + - **Usage**: + - `{{"name": "custom_tool", "args": {{"parameter": "value"}}}}` + - Example: Custom analytics tool. + - Command: `{{"name": "custom_tool", "args": {{"data": "analyze this data"}}}}` + + + ### Usage Examples + - **Example 1**: Retrieving Weather Information + ```json + {tool_usage_browser} + ``` + + - **Example 2**: System Check via Terminal + ```json + {tool_usage_terminal} + ``` + + - **Example 3**: Combined Browser and Terminal Usage + ```json + {browser_and_terminal_tool} + ``` + + - **Example 4**: Combined Browser, Terminal, and Calculator Usage + ```json + {browser_and_terminal_tool_two} + ``` + - 1. Continuously review and analyze your actions to ensure you are performing to the best of your abilities. - 2. Constructively self-criticize your big-picture behavior constantly. - 3. Reflect on past decisions and strategies to refine your approach. - 4. Every command has a cost, so be smart and efficient. Aim to complete tasks in the least number of steps. + + ### Next Steps + - Determine the appropriate tool for the task at hand. + - Format your command according to the examples provided. + - Execute the command and evaluate the results based on the expected outcome. + - Document any issues or challenges faced during the tool usage. + - Always output the results in the specified format: JSON in markdown code blocks. - You should only respond in JSON format as described below Response Format, you will respond only in markdown format within 6 backticks. The JSON will be in markdown format. - ``` - {response_json} - ``` + ###### Tools Available - Ensure the response can be parsed by Python json.loads - System: The current time and date is {time} - System: This reminds you of these events from your past: - [{memory}] + {tool_docs} - Human: Determine which next command to use, and respond using the format specified above: + This SOP is designed to guide you through the structured and effective use of tools. By adhering to this protocol, you will enhance your productivity and accuracy in task execution. """ - return str(out) + return prompt diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index 9b516111..14904d14 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -6,7 +6,7 @@ import random import sys import time import uuid -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple import yaml from loguru import logger @@ -17,7 +17,6 @@ from swarms.prompts.agent_system_prompts import AGENT_SYSTEM_PROMPT_3 from swarms.prompts.multi_modal_autonomous_instruction_prompt import ( MULTI_MODAL_AUTO_AGENT_SYSTEM_PROMPT_1, ) -from swarms.prompts.worker_prompt import worker_tools_sop_promp from swarms.structs.conversation import Conversation from swarms.tools.tool import BaseTool from swarms.utils.code_interpreter import SubprocessCodeInterpreter @@ -25,8 +24,8 @@ from swarms.utils.data_to_text import data_to_text from swarms.utils.parse_code import extract_code_from_markdown from swarms.utils.pdf_to_text import pdf_to_text from swarms.tools.exec_tool import execute_tool_by_name -from swarms.tools.function_util import process_tool_docs from swarms.tools.code_executor import CodeExecutor +from swarms.prompts.worker_prompt import tool_usage_worker_prompt # Utils @@ -172,7 +171,7 @@ class Agent: agent_name: str = "swarm-worker-01", agent_description: str = None, system_prompt: str = AGENT_SYSTEM_PROMPT_3, - tools: Union[List[BaseTool]] = None, + tools: List[BaseTool] = [], dynamic_temperature_enabled: Optional[bool] = False, sop: Optional[str] = None, sop_list: Optional[List[str]] = None, @@ -210,6 +209,7 @@ class Agent: custom_exit_command: Optional[str] = "exit", sentiment_analyzer: Optional[Callable] = None, limit_tokens_from_string: Optional[Callable] = None, + custom_tools_prompt: Optional[Callable] = None, *args, **kwargs, ): @@ -318,21 +318,21 @@ class Agent: # If tools are provided then set the tool prompt by adding to sop if self.tools: - tools_prompt = worker_tools_sop_promp( - name=self.agent_name, - memory=self.short_memory.return_history_as_string(), - ) + if custom_tools_prompt is not None: + tools_prompt = custom_tools_prompt(tools=self.tools) - # Append the tools prompt to the short_term_memory - self.short_memory.add( - role=self.agent_name, content=tools_prompt - ) + self.short_memory.add( + role=self.agent_name, content=tools_prompt + ) - # And, add the tool documentation to the memory - for tool in self.tools: - tool_docs = process_tool_docs(tool) + else: + tools_prompt = tool_usage_worker_prompt( + tools=self.tools + ) + + # Append the tools prompt to the short_term_memory self.short_memory.add( - role=self.agent_name, content=tool_docs + role=self.agent_name, content=tools_prompt ) # If the long term memory is provided then set the long term memory prompt @@ -713,10 +713,11 @@ class Agent: break # Exit the loop if all retry attempts fail # Check stopping conditions - if self.stopping_token in response: - break + if self.stopping_token is not None: + if self.stopping_token in response: + break elif ( - self.stopping_condition + self.stopping_condition is not None and self._check_stopping_condition(response) ): break diff --git a/tool.py b/tool.py new file mode 100644 index 00000000..0f0b4a80 --- /dev/null +++ b/tool.py @@ -0,0 +1,57 @@ +from swarms import Agent, Anthropic, tool + +# Model +llm = Anthropic( + temperature=0.1, +) + +""" +How to create tools: + +1. Define a function that takes the required arguments with documentation and type hints. +2. Add the `@tool` decorator to the function. +3. Add the function to the `tools` list in the `Agent` class. +""" + + +# Tools +# Browser tools +@tool +def browser(query: str): + """ + Opens a web browser and searches for the given query on Google. + + Args: + query (str): The search query. + + Returns: + str: A message indicating that the search is being performed. + """ + import webbrowser + + url = f"https://www.google.com/search?q={query}" + webbrowser.open(url) + return f"Searching for {query} in the browser." + + +# Agent +agent = Agent( + agent_name="Devin", + system_prompt=( + "Autonomous agent that can interact with humans and other" + " agents. Be Helpful and Kind. Use the tools provided to" + " assist the user. Return all code in markdown format." + ), + llm=llm, + max_loops="auto", + autosave=True, + dashboard=False, + verbose=True, + stopping_token="", + interactive=True, + tools=[browser], +) + +# Run the agent +out = agent.run("what's the weather in Miami?") +print(out) From 9c75781cf632e4c99ed04ac7144386a5c76d63a1 Mon Sep 17 00:00:00 2001 From: Kye Date: Wed, 17 Apr 2024 16:43:01 -0400 Subject: [PATCH 2/2] [CLEANUP] --- devin.py => playground/agents/devin.py | 0 .../creation_engine/omni_model_agent.py | 82 +++++++++++++++++++ tool.py => playground/youtube/tool.py | 0 pyproject.toml | 2 +- scripts/log_cleanup.py | 12 +-- scripts/log_cleanup.sh | 10 +++ swarms/models/open_router.py | 75 +++++++++++++++++ swarms/structs/agent.py | 7 +- swarms/tools/exec_tool.py | 6 ++ 9 files changed, 186 insertions(+), 8 deletions(-) rename devin.py => playground/agents/devin.py (100%) create mode 100644 playground/creation_engine/omni_model_agent.py rename tool.py => playground/youtube/tool.py (100%) create mode 100755 scripts/log_cleanup.sh diff --git a/devin.py b/playground/agents/devin.py similarity index 100% rename from devin.py rename to playground/agents/devin.py diff --git a/playground/creation_engine/omni_model_agent.py b/playground/creation_engine/omni_model_agent.py new file mode 100644 index 00000000..b261c2f7 --- /dev/null +++ b/playground/creation_engine/omni_model_agent.py @@ -0,0 +1,82 @@ +from swarms import Agent, Anthropic, tool + +# Model +llm = Anthropic( + temperature=0.1, +) + + +# Tools +@tool +def text_to_video(task: str): + """ + Converts a given text task into an animated video. + + Args: + task (str): The text task to be converted into a video. + + Returns: + str: The path to the exported GIF file. + """ + import torch + from diffusers import ( + AnimateDiffPipeline, + MotionAdapter, + EulerDiscreteScheduler, + ) + from diffusers.utils import export_to_gif + from huggingface_hub import hf_hub_download + from safetensors.torch import load_file + + device = "cuda" + dtype = torch.float16 + + step = 4 # Options: [1,2,4,8] + repo = "ByteDance/AnimateDiff-Lightning" + ckpt = f"animatediff_lightning_{step}step_diffusers.safetensors" + base = ( # Choose to your favorite base model. + "emilianJR/epiCRealism" + ) + + adapter = MotionAdapter().to(device, dtype) + adapter.load_state_dict( + load_file(hf_hub_download(repo, ckpt), device=device) + ) + pipe = AnimateDiffPipeline.from_pretrained( + base, motion_adapter=adapter, torch_dtype=dtype + ).to(device) + pipe.scheduler = EulerDiscreteScheduler.from_config( + pipe.scheduler.config, + timestep_spacing="trailing", + beta_schedule="linear", + ) + + output = pipe( + prompt=task, guidance_scale=1.0, num_inference_steps=step + ) + out = export_to_gif(output.frames[0], "animation.gif") + return out + + +# Agent +agent = Agent( + agent_name="Devin", + system_prompt=( + "Autonomous agent that can interact with humans and other" + " agents. Be Helpful and Kind. Use the tools provided to" + " assist the user. Return all code in markdown format." + ), + llm=llm, + max_loops="auto", + autosave=True, + dashboard=False, + streaming_on=True, + verbose=True, + stopping_token="", + interactive=True, + tools=[text_to_video], +) + +# Run the agent +out = agent("Create a vide of a girl coding AI wearing hijab") +print(out) diff --git a/tool.py b/playground/youtube/tool.py similarity index 100% rename from tool.py rename to playground/youtube/tool.py diff --git a/pyproject.toml b/pyproject.toml index b9c2fc1a..9e4551e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "swarms" -version = "4.8.1" +version = "4.8.2" description = "Swarms - Pytorch" license = "MIT" authors = ["Kye Gomez "] diff --git a/scripts/log_cleanup.py b/scripts/log_cleanup.py index ad3da11b..368ceb63 100644 --- a/scripts/log_cleanup.py +++ b/scripts/log_cleanup.py @@ -2,8 +2,8 @@ import os import shutil # Create a new directory for the log files if it doesn't exist -if not os.path.exists("artifacts"): - os.makedirs("artifacts") +if not os.path.exists("artifacts_two"): + os.makedirs("artifacts_two") # Walk through the current directory for dirpath, dirnames, filenames in os.walk("."): @@ -12,10 +12,10 @@ for dirpath, dirnames, filenames in os.walk("."): if filename.endswith(".log"): # Construct the full file path file_path = os.path.join(dirpath, filename) - # Move the log file to the 'artifacts' directory - shutil.move(file_path, "artifacts") + # Move the log file to the 'artifacts_two' directory + shutil.move(file_path, "artifacts_two") print( - "Moved all log files into the 'artifacts' directory and deleted" - " their original location." + "Moved all log files into the 'artifacts_two' directory and" + " deleted their original location." ) diff --git a/scripts/log_cleanup.sh b/scripts/log_cleanup.sh new file mode 100755 index 00000000..aa0bb83c --- /dev/null +++ b/scripts/log_cleanup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Create the new directory if it doesn't exist +sudo mkdir -p /artifacts_logs + +# Find all .log files in the root directory and its subdirectories +find / -name "*.log" -print0 | while IFS= read -r -d '' file; do + # Use sudo to move the file to the new directory + sudo mv "$file" /artifacts_logs/ +done \ No newline at end of file diff --git a/swarms/models/open_router.py b/swarms/models/open_router.py index e69de29b..7aff0aca 100644 --- a/swarms/models/open_router.py +++ b/swarms/models/open_router.py @@ -0,0 +1,75 @@ +from swarms.models.base_llm import AbstractLLM +from pydantic import BaseModel +from typing import List, Dict +import openai + + +class OpenRouterRequest(BaseModel): + model: str + messages: List[Dict[str, str]] = [] + + +class OpenRouterChat(AbstractLLM): + """ + A class representing an OpenRouter chat model. + + Args: + model_name (str): The name of the OpenRouter model. + base_url (str, optional): The base URL for the OpenRouter API. Defaults to "https://openrouter.ai/api/v1/chat/completions". + openrouter_api_key (str, optional): The API key for accessing the OpenRouter API. Defaults to None. + system_prompt (str, optional): The system prompt for the chat model. Defaults to None. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Attributes: + model_name (str): The name of the OpenRouter model. + base_url (str): The base URL for the OpenRouter API. + openrouter_api_key (str): The API key for accessing the OpenRouter API. + system_prompt (str): The system prompt for the chat model. + + Methods: + run(task, *args, **kwargs): Runs the chat model with the given task. + + """ + + def __init__( + self, + model_name: str, + base_url: str = "https://openrouter.ai/api/v1/chat/completions", + openrouter_api_key: str = None, + system_prompt: str = None, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.model_name = model_name + self.base_url = base_url + self.openrouter_api_key = openrouter_api_key + self.system_prompt = system_prompt + + openai.api_base = "https://openrouter.ai/api/v1" + openai.api_key = openrouter_api_key + + def run(self, task: str, *args, **kwargs) -> str: + """ + Runs the chat model with the given task. + + Args: + task (str): The user's task for the chat model. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + str: The response generated by the chat model. + + """ + response = openai.ChatCompletion.create( + model=self.model_name, + messages=[ + {"role": "system", "content": self.system_prompt}, + {"role": "user", "content": task}, + ] + * args, + **kwargs, + ) + return response.choices[0].message.text diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index 14904d14..fc7a023e 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -614,7 +614,7 @@ class Agent: else (task_prompt, img, *args) ) response = self.llm(*response_args, **kwargs) - print(response) + # print(response) self.short_memory.add( role=self.agent_name, content=response ) @@ -696,6 +696,11 @@ class Agent: content=sentiment, ) + if self.streaming: + self.streaming(response) + else: + print(response) + success = True # Mark as successful to exit the retry loop except Exception as e: diff --git a/swarms/tools/exec_tool.py b/swarms/tools/exec_tool.py index 2190398a..558cb9b5 100644 --- a/swarms/tools/exec_tool.py +++ b/swarms/tools/exec_tool.py @@ -9,6 +9,8 @@ from pydantic import ValidationError from swarms.tools.tool import BaseTool +from swarms.utils.loguru_logger import logger + class AgentAction(NamedTuple): """Action returned by AgentOutputParser.""" @@ -97,6 +99,9 @@ def execute_tool_by_name( # Get command name and arguments action = output_parser.parse(text) tools = {t.name: t for t in tools} + + logger.info(f"Tools available: {tools}") + if action.name == stop_token: return action.args["response"] if action.name in tools: @@ -109,6 +114,7 @@ def execute_tool_by_name( with concurrent.futures.ThreadPoolExecutor() as executor: futures = [] for tool_name in tool_names: + logger.info(f"Executing tool: {tool_name}") futures.append( executor.submit( tools[tool_name].run, action.args