diff --git a/README.md b/README.md index 43407822..aa8822e8 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,56 @@ generated_data = agent.run(task) print(generated_data) +``` + + +### `Worker` +The `Worker` is a simple all-in-one agent equipped with an LLM, tools, and RAG. Get started below: + +✅ Plug in and Play LLM. Utilize any LLM from anywhere and any framework + +✅ Reliable RAG: Utilizes FAISS for efficient RAG but it's modular so you can use any DB. + +✅ Multi-Step Parallel Function Calling: Use any tool + +```python +# Importing necessary modules +import os +from dotenv import load_dotenv +from swarms import Worker, OpenAIChat, tool + +# Loading environment variables from .env file +load_dotenv() + +# Retrieving the OpenAI API key from environment variables +api_key = os.getenv("OPENAI_API_KEY") + + +# Create a tool +@tool +def search_api(query: str): + pass + + +# Creating a Worker instance +worker = Worker( + name="My Worker", + role="Worker", + human_in_the_loop=False, + tools=[search_api], + temperature=0.5, + llm=OpenAIChat(openai_api_key=api_key), +) + +# Running the worker with a prompt +out = worker.run( + "Hello, how are you? Create an image of how your are doing!" +) + +# Printing the output +print(out) + + ``` ------ @@ -774,6 +824,115 @@ out = llm.run(task=task, img=img) print(out) ``` +### `GPT4Vision` +```python +from swarms import GPT4VisionAPI + +# Initialize with default API key and custom max_tokens +api = GPT4VisionAPI(max_tokens=1000) + +# Define the task and image URL +task = "Describe the scene in the image." +img = "https://i.imgur.com/4P4ZRxU.jpeg" + +# Run the GPT-4 Vision model +response = api.run(task, img) + +# Print the model's response +print(response) +``` + +### `QwenVLMultiModal` +A radically simple interface for QwenVLMultiModal comes complete with Quantization to turn it on just set quantize to true! + +```python +from swarms import QwenVLMultiModal + +# Instantiate the QwenVLMultiModal model +model = QwenVLMultiModal( + model_name="Qwen/Qwen-VL-Chat", + device="cuda", + quantize=True, +) + +# Run the model +response = model( + "Hello, how are you?", "https://example.com/image.jpg" +) + +# Print the response +print(response) + + +``` + + +### `Kosmos` +- Multi-Modal Model from microsoft! + +```python +from swarms import Kosmos + +# Initialize the model +model = Kosmos() + +# Generate +out = model.run("Analyze the reciepts in this image", "docs.jpg") + +# Print the output +print(out) + +``` + + +### `Idefics` +- Multi-Modal model from Huggingface team! + +```python +# Import the idefics model from the swarms.models module +from swarms.models import Idefics + +# Create an instance of the idefics model +model = Idefics() + +# Define user input with an image URL and chat with the model +user_input = ( + "User: What is in this image?" + " https://upload.wikimedia.org/wikipedia/commons/8/86/Id%C3%A9fix.JPG" +) +response = model.chat(user_input) +print(response) + +# Define another user input with an image URL and chat with the model +user_input = ( + "User: And who is that?" + " https://static.wikia.nocookie.net/asterix/images/2/25/R22b.gif/revision/latest?cb=20110815073052" +) +response = model.chat(user_input) +print(response) + +# Set the checkpoint of the model to "new_checkpoint" +model.set_checkpoint("new_checkpoint") + +# Set the device of the model to "cpu" +model.set_device("cpu") + +# Set the maximum length of the chat to 200 +model.set_max_length(200) + +# Clear the chat history of the model +model.clear_chat_history() + + +``` + +## Radically Simple AI Model APIs +We provide a vast array of language and multi-modal model APIs for you to generate text, images, music, speech, and even videos. Get started below: + + + +----- + ### `Anthropic` ```python @@ -850,23 +1009,6 @@ print(image_url) ``` -### `GPT4Vision` -```python -from swarms import GPT4VisionAPI - -# Initialize with default API key and custom max_tokens -api = GPT4VisionAPI(max_tokens=1000) - -# Define the task and image URL -task = "Describe the scene in the image." -img = "https://i.imgur.com/4P4ZRxU.jpeg" - -# Run the GPT-4 Vision model -response = api.run(task, img) - -# Print the model's response -print(response) -``` ### Text to Video with `ZeroscopeTTV` @@ -888,7 +1030,7 @@ print(video_path) ``` -### ModelScope + -### `QwenVLMultiModal` -A radically simple interface for QwenVLMultiModal comes complete with Quantization to turn it on just set quantize to true! - -```python -from swarms import QwenVLMultiModal - -# Instantiate the QwenVLMultiModal model -model = QwenVLMultiModal( - model_name="Qwen/Qwen-VL-Chat", - device="cuda", - quantize=True, -) - -# Run the model -response = model( - "Hello, how are you?", "https://example.com/image.jpg" -) - -# Print the response -print(response) - - -``` ---- diff --git a/docs/corporate/data_room.md b/docs/corporate/data_room.md index 550e8f94..946f209f 100644 --- a/docs/corporate/data_room.md +++ b/docs/corporate/data_room.md @@ -80,23 +80,16 @@ Swarms is an open source framework for developers in python to enable seamless, [Here is the official Swarms Github Page:](https://github.com/kyegomez/swarms) ### Product Growth Metrics +| Name | Description | Link | +|----------------------------------|---------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------| +| Total Downloads of all time | Total number of downloads for the product over its entire lifespan. | [![Downloads](https://static.pepy.tech/badge/swarms)](https://pepy.tech/project/swarms) | +| Downloads this month | Number of downloads for the product in the current month. | [![Downloads](https://static.pepy.tech/badge/swarms/month)](https://pepy.tech/project/swarms) | +| Total Downloads this week | Total number of downloads for the product in the current week. | [![Downloads](https://static.pepy.tech/badge/swarms/week)](https://pepy.tech/project/swarms) | +| Github Forks | Number of times the product's codebase has been copied for optimization, contribution, or usage. | [![GitHub forks](https://img.shields.io/github/forks/kyegomez/swarms)](https://github.com/kyegomez/swarms/network) | +| Github Stars | Number of users who have 'liked' the project. | [![GitHub stars](https://img.shields.io/github/stars/kyegomez/swarms)](https://github.com/kyegomez/swarms/stargazers) | +| Pip Module Metrics | Various project statistics such as watchers, number of contributors, date repository was created, and more. | [CLICK HERE](https://libraries.io/github/kyegomez/swarms) | +| Contribution Based Statistics | Statistics like number of contributors, lines of code changed, etc. | [HERE](https://github.com/kyegomez/swarms/graphs/contributors) | +| Github Community insights | Insights into the Github community around the product. | [Github Community insights](https://github.com/kyegomez/swarms/graphs/community) | +| Github Traffic Metrics | Metrics related to traffic, such as views and clones on Github. | [Github Traffic Metrics](https://github.com/kyegomez/swarms/graphs/traffic) | +| Issues with the framework | Current open issues for the product on Github. | [![GitHub issues](https://img.shields.io/github/issues/kyegomez/swarms)](https://github.com/kyegomez/swarms/issues) | -- Total Downloads of all time: [![GitHub issues](https://img.shields.io/github/issues/kyegomez/swarms)](https://github.com/kyegomez/swarms/issues) - -- Click here for Downloads this month: [![Downloads](https://static.pepy.tech/badge/swarms/month)](https://pepy.tech/project/swarms) - -- Total Downloads this week: [![GitHub issues](https://img.shields.io/github/issues/kyegomez/swarms)](https://github.com/kyegomez/swarms/issues) - -- Click here for Forks which represent the number of times a user has copied the entire codebase for optimization, contribution, or usage. [![GitHub forks](https://img.shields.io/github/forks/kyegomez/swarms)](https://github.com/kyegomez/swarms/network) - -- Stars are the number of people that have liked our project, click here for more: [![GitHub stars](https://img.shields.io/github/stars/kyegomez/swarms)](https://github.com/kyegomez/swarms/stargazers) - -- Various Project Statistics such as watchers, number of contributors, date repository was created and much more. [CLICK HERE](https://libraries.io/github/kyegomez/swarms) - -- Contribution Based Statistics such as number of contributors, number of lines of code changed, and much more [HERE](https://github.com/kyegomez/swarms/graphs/contributors) - -- [Github Community insights](https://github.com/kyegomez/swarms/graphs/community) - -- [Github Traffic Metrics](https://github.com/kyegomez/swarms/graphs/traffic) - -- Issues with the framework or Github Issues: [![GitHub issues](https://img.shields.io/github/issues/kyegomez/swarms)](https://github.com/kyegomez/swarms/issues) \ No newline at end of file diff --git a/playground/agents/worker_example.py b/playground/agents/worker_example.py index ceead3a9..9e215e83 100644 --- a/playground/agents/worker_example.py +++ b/playground/agents/worker_example.py @@ -1,12 +1,16 @@ +# Importing necessary modules import os from dotenv import load_dotenv from swarms.agents.worker_agent import Worker from swarms import OpenAIChat +# Loading environment variables from .env file load_dotenv() +# Retrieving the OpenAI API key from environment variables api_key = os.getenv("OPENAI_API_KEY") +# Creating a Worker instance worker = Worker( name="My Worker", role="Worker", @@ -14,9 +18,13 @@ worker = Worker( tools=[], temperature=0.5, llm=OpenAIChat(openai_api_key=api_key), + verbose=True, ) +# Running the worker with a prompt out = worker.run( "Hello, how are you? Create an image of how your are doing!" ) + +# Printing the output print(out) diff --git a/playground/demos/fof/langchain.py b/playground/demos/fof/langchain.py new file mode 100644 index 00000000..dd6d7083 --- /dev/null +++ b/playground/demos/fof/langchain.py @@ -0,0 +1,4 @@ +""" +This tutorial shows you how to integrate swarms with Langchain + +""" diff --git a/playground/models/idefics.py b/playground/models/idefics.py index 39d6f4eb..ea36ba77 100644 --- a/playground/models/idefics.py +++ b/playground/models/idefics.py @@ -1,7 +1,10 @@ -from swarms.models import idefics +# Import the idefics model from the swarms.models module +from swarms.models import Idefics -model = idefics() +# Create an instance of the idefics model +model = Idefics() +# Define user input with an image URL and chat with the model user_input = ( "User: What is in this image?" " https://upload.wikimedia.org/wikipedia/commons/8/86/Id%C3%A9fix.JPG" @@ -9,6 +12,7 @@ user_input = ( response = model.chat(user_input) print(response) +# Define another user input with an image URL and chat with the model user_input = ( "User: And who is that?" " https://static.wikia.nocookie.net/asterix/images/2/25/R22b.gif/revision/latest?cb=20110815073052" @@ -16,7 +20,14 @@ user_input = ( response = model.chat(user_input) print(response) +# Set the checkpoint of the model to "new_checkpoint" model.set_checkpoint("new_checkpoint") + +# Set the device of the model to "cpu" model.set_device("cpu") + +# Set the maximum length of the chat to 200 model.set_max_length(200) + +# Clear the chat history of the model model.clear_chat_history() diff --git a/playground/models/kosmos.py b/playground/models/kosmos.py new file mode 100644 index 00000000..dbfd108f --- /dev/null +++ b/playground/models/kosmos.py @@ -0,0 +1,10 @@ +from swarms import Kosmos + +# Initialize the model +model = Kosmos() + +# Generate +out = model.run("Analyze the reciepts in this image", "docs.jpg") + +# Print the output +print(out) diff --git a/pyproject.toml b/pyproject.toml index fdd127c3..31b72863 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "swarms" -version = "3.7.9" +version = "3.8.5" description = "Swarms - Pytorch" license = "MIT" authors = ["Kye Gomez "] @@ -24,7 +24,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.6.1" torch = "2.1.1" -transformers = "4.36.2" +transformers = "4.37.1" openai = "0.28.0" langchain = "0.0.333" asyncio = "3.4.3" diff --git a/swarms/agents/__init__.py b/swarms/agents/__init__.py index 4b786bf4..461baa16 100644 --- a/swarms/agents/__init__.py +++ b/swarms/agents/__init__.py @@ -15,6 +15,7 @@ from swarms.agents.stopping_conditions import ( ) from swarms.agents.tool_agent import ToolAgent from swarms.agents.worker_agent import Worker +from swarms.agents.agent_wrapper import agent_wrapper __all__ = [ "AbstractAgent", @@ -32,4 +33,5 @@ __all__ = [ "check_exit", "check_end", "Worker", + "agent_wrapper", ] diff --git a/swarms/agents/agent_wrapper.py b/swarms/agents/agent_wrapper.py new file mode 100644 index 00000000..738f599d --- /dev/null +++ b/swarms/agents/agent_wrapper.py @@ -0,0 +1,26 @@ +from swarms.structs.agent import Agent + + +def agent_wrapper(ClassToWrap): + """ + This function takes a class 'ClassToWrap' and returns a new class that + inherits from both 'ClassToWrap' and 'Agent'. The new class overrides + the '__init__' method of 'Agent' to call the '__init__' method of 'ClassToWrap'. + + Args: + ClassToWrap (type): The class to be wrapped and made to inherit from 'Agent'. + + Returns: + type: The new class that inherits from both 'ClassToWrap' and 'Agent'. + """ + + class WrappedClass(ClassToWrap, Agent): + def __init__(self, *args, **kwargs): + try: + Agent.__init__(self, *args, **kwargs) + ClassToWrap.__init__(self, *args, **kwargs) + except Exception as e: + print(f"Error initializing WrappedClass: {e}") + raise e + + return WrappedClass diff --git a/swarms/agents/worker_agent.py b/swarms/agents/worker_agent.py index 8ed7d0d3..d254acef 100644 --- a/swarms/agents/worker_agent.py +++ b/swarms/agents/worker_agent.py @@ -51,6 +51,7 @@ class Worker: tools: List[Any] = None, embedding_size: int = 1536, search_kwargs: dict = {"k": 8}, + verbose: bool = False, *args, **kwargs, ): @@ -64,6 +65,7 @@ class Worker: self.tools = tools self.embedding_size = embedding_size self.search_kwargs = search_kwargs + self.verbose = verbose self.setup_tools(external_tools) self.setup_memory() @@ -163,7 +165,7 @@ class Worker: # @log_decorator @error_decorator @timing_decorator - def run(self, task: str = None, *args, **kwargs): + def run(self, task: str = None, img=None, *args, **kwargs): """ Run the autonomous agent on a given task. diff --git a/swarms/memory/__init__.py b/swarms/memory/__init__.py index d2eed0d5..1d52d718 100644 --- a/swarms/memory/__init__.py +++ b/swarms/memory/__init__.py @@ -1,11 +1,11 @@ -from swarms.memory.base_vectordb import VectorDatabase +from swarms.memory.base_vectordb import AbstractDatabase from swarms.memory.short_term_memory import ShortTermMemory from swarms.memory.sqlite import SQLiteDB from swarms.memory.weaviate_db import WeaviateDB from swarms.memory.visual_memory import VisualShortTermMemory __all__ = [ - "VectorDatabase", + "AbstractDatabase", "ShortTermMemory", "SQLiteDB", "WeaviateDB", diff --git a/swarms/memory/base_vectordatabase.py b/swarms/memory/base_vectordatabase.py new file mode 100644 index 00000000..06f42007 --- /dev/null +++ b/swarms/memory/base_vectordatabase.py @@ -0,0 +1,141 @@ +from abc import ABC, abstractmethod + + +class AbstractVectorDatabase(ABC): + """ + Abstract base class for a database. + + This class defines the interface for interacting with a database. + Subclasses must implement the abstract methods to provide the + specific implementation details for connecting to a database, + executing queries, and performing CRUD operations. + + """ + + @abstractmethod + def connect(self): + """ + Connect to the database. + + This method establishes a connection to the database. + + """ + + pass + + @abstractmethod + def close(self): + """ + Close the database connection. + + This method closes the connection to the database. + + """ + + pass + + @abstractmethod + def query(self, query: str): + """ + Execute a database query. + + This method executes the given query on the database. + + Parameters: + query (str): The query to be executed. + + """ + + pass + + @abstractmethod + def fetch_all(self): + """ + Fetch all rows from the result set. + + This method retrieves all rows from the result set of a query. + + Returns: + list: A list of dictionaries representing the rows. + + """ + + pass + + @abstractmethod + def fetch_one(self): + """ + Fetch one row from the result set. + + This method retrieves one row from the result set of a query. + + Returns: + dict: A dictionary representing the row. + + """ + + pass + + @abstractmethod + def add(self, doc: str): + """ + Add a new record to the database. + + This method adds a new record to the specified table in the database. + + Parameters: + table (str): The name of the table. + data (dict): A dictionary representing the data to be added. + + """ + + pass + + @abstractmethod + def get(self, query: str): + """ + Get a record from the database. + + This method retrieves a record from the specified table in the database based on the given ID. + + Parameters: + table (str): The name of the table. + id (int): The ID of the record to be retrieved. + + Returns: + dict: A dictionary representing the retrieved record. + + """ + + pass + + @abstractmethod + def update(self, doc): + """ + Update a record in the database. + + This method updates a record in the specified table in the database based on the given ID. + + Parameters: + table (str): The name of the table. + id (int): The ID of the record to be updated. + data (dict): A dictionary representing the updated data. + + """ + + pass + + @abstractmethod + def delete(self, message): + """ + Delete a record from the database. + + This method deletes a record from the specified table in the database based on the given ID. + + Parameters: + table (str): The name of the table. + id (int): The ID of the record to be deleted. + + """ + + pass diff --git a/swarms/memory/base_vectordb.py b/swarms/memory/base_vectordb.py deleted file mode 100644 index 841c6147..00000000 --- a/swarms/memory/base_vectordb.py +++ /dev/null @@ -1,58 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Dict - - -class VectorDatabase(ABC): - @abstractmethod - def add( - self, vector: Dict[str, Any], metadata: Dict[str, Any] - ) -> None: - """ - add a vector into the database. - - Args: - vector (Dict[str, Any]): The vector to add. - metadata (Dict[str, Any]): Metadata associated with the vector. - """ - pass - - @abstractmethod - def query(self, text: str, num_results: int) -> Dict[str, Any]: - """ - Query the database for vectors similar to the given vector. - - Args: - text (Dict[str, Any]): The vector to compare against. - num_results (int): The number of similar vectors to return. - - Returns: - Dict[str, Any]: The most similar vectors and their associated metadata. - """ - pass - - @abstractmethod - def delete(self, vector_id: str) -> None: - """ - Delete a vector from the database. - - Args: - vector_id (str): The ID of the vector to delete. - """ - pass - - @abstractmethod - def update( - self, - vector_id: str, - vector: Dict[str, Any], - metadata: Dict[str, Any], - ) -> None: - """ - Update a vector in the database. - - Args: - vector_id (str): The ID of the vector to update. - vector (Dict[str, Any]): The new vector. - metadata (Dict[str, Any]): The new metadata. - """ - pass diff --git a/swarms/memory/pinecone.py b/swarms/memory/pinecone.py index 164cb334..bf073d3e 100644 --- a/swarms/memory/pinecone.py +++ b/swarms/memory/pinecone.py @@ -1,12 +1,12 @@ from typing import Optional -from swarms.memory.base_vectordb import VectorDatabase +from swarms.memory.base_vectordb import AbstractDatabase import pinecone from attr import define, field from swarms.utils.hash import str_to_hash @define -class PineconeDB(VectorDatabase): +class PineconeDB(AbstractDatabase): """ PineconeDB is a vector storage driver that uses Pinecone as the underlying storage engine. diff --git a/swarms/memory/sqlite.py b/swarms/memory/sqlite.py index eed4ee2c..2c4f2740 100644 --- a/swarms/memory/sqlite.py +++ b/swarms/memory/sqlite.py @@ -1,5 +1,5 @@ from typing import List, Tuple, Any, Optional -from swarms.memory.base_vectordb import VectorDatabase +from swarms.memory.base_vectordb import AbstractDatabase try: import sqlite3 @@ -9,7 +9,7 @@ except ImportError: ) -class SQLiteDB(VectorDatabase): +class SQLiteDB(AbstractDatabase): """ A reusable class for SQLite database operations with methods for adding, deleting, updating, and querying data. diff --git a/swarms/memory/weaviate_db.py b/swarms/memory/weaviate_db.py index 0c0b09a2..fec1199e 100644 --- a/swarms/memory/weaviate_db.py +++ b/swarms/memory/weaviate_db.py @@ -4,7 +4,7 @@ Weaviate API Client from typing import Any, Dict, List, Optional -from swarms.memory.base_vectordb import VectorDatabase +from swarms.memory.base_vectordb import AbstractDatabase try: import weaviate @@ -12,7 +12,7 @@ except ImportError: print("pip install weaviate-client") -class WeaviateDB(VectorDatabase): +class WeaviateDB(AbstractDatabase): """ Weaviate API Client diff --git a/swarms/models/__init__.py b/swarms/models/__init__.py index 635124a6..dfeb9cfe 100644 --- a/swarms/models/__init__.py +++ b/swarms/models/__init__.py @@ -47,6 +47,9 @@ from swarms.models.ultralytics_model import ( from swarms.models.vip_llava import VipLlavaMultiModal # noqa: E402 from swarms.models.llava import LavaMultiModal # noqa: E402 from swarms.models.qwen import QwenVLMultiModal # noqa: E402 +from swarms.models.clipq import CLIPQ # noqa: E402 +from swarms.models.kosmos_two import Kosmos # noqa: E402 +from swarms.models.fuyu import Fuyu # noqa: E402 # from swarms.models.dalle3 import Dalle3 # from swarms.models.distilled_whisperx import DistilWhisperModel # noqa: E402 @@ -78,7 +81,6 @@ __all__ = [ "Zephyr", "BaseMultiModalModel", "Idefics", - # "Kosmos", "Vilt", "Nougat", "LayoutLMDocumentQA", @@ -101,13 +103,13 @@ __all__ = [ "AudioModality", "VideoModality", "MultimodalData", - # "CogAgent", - # "ModelScopePipeline", - # "ModelScopeAutoModel", "TogetherLLM", "TimmModel", "UltralyticsModel", "VipLlavaMultiModal", "LavaMultiModal", "QwenVLMultiModal", + "CLIPQ", + "Kosmos", + "Fuyu", ] diff --git a/swarms/models/anthropic.py b/swarms/models/anthropic.py index adffe49d..0e4690f9 100644 --- a/swarms/models/anthropic.py +++ b/swarms/models/anthropic.py @@ -29,9 +29,7 @@ from langchain.schema.language_model import BaseLanguageModel from langchain.schema.output import GenerationChunk from langchain.schema.prompt import PromptValue from langchain.utils import ( - check_package_version, get_from_dict_or_env, - get_pydantic_field_names, ) from packaging.version import parse from requests import HTTPError, Response diff --git a/swarms/models/clipq.py b/swarms/models/clipq.py new file mode 100644 index 00000000..7e49e74a --- /dev/null +++ b/swarms/models/clipq.py @@ -0,0 +1,183 @@ +from io import BytesIO + +import requests +import torch +from PIL import Image +from torchvision.transforms import GaussianBlur +from transformers import CLIPModel, CLIPProcessor + + +class CLIPQ: + """ + ClipQ is an CLIQ based model that can be used to generate captions for images. + + + Attributes: + model_name (str): The name of the model to be used. + query_text (str): The query text to be used for the model. + + Args: + model_name (str): The name of the model to be used. + query_text (str): The query text to be used for the model. + + + + + """ + + def __init__( + self, + model_name: str = "openai/clip-vit-base-patch16", + query_text: str = "A photo ", + *args, + **kwargs, + ): + self.model = CLIPModel.from_pretrained( + model_name, *args, **kwargs + ) + self.processor = CLIPProcessor.from_pretrained(model_name) + self.query_text = query_text + + def fetch_image_from_url(self, url="https://picsum.photos/800"): + """Fetches an image from the given url""" + response = requests.get(url) + if response.status_code != 200: + raise Exception("Failed to fetch an image") + image = Image.open(BytesIO(response.content)) + return image + + def load_image_from_path(self, path): + """Loads an image from the given path""" + return Image.open(path) + + def split_image( + self, image, h_splits: int = 2, v_splits: int = 2 + ): + """Splits the given image into h_splits x v_splits parts""" + width, height = image.size + w_step, h_step = width // h_splits, height // v_splits + slices = [] + + for i in range(v_splits): + for j in range(h_splits): + slice = image.crop( + ( + j * w_step, + i * h_step, + (j + 1) * w_step, + (i + 1) * h_step, + ) + ) + slices.append(slice) + return slices + + def get_vectors( + self, + image, + h_splits: int = 2, + v_splits: int = 2, + ): + """Gets the vectors for the given image""" + slices = self.split_image(image, h_splits, v_splits) + vectors = [] + + for slice in slices: + inputs = self.processor( + text=self.query_text, + images=slice, + return_tensors="pt", + padding=True, + ) + outputs = self.model(**inputs) + vectors.append( + outputs.image_embeds.squeeze().detach().numpy() + ) + return vectors + + def run_from_url( + self, + url: str = "https://picsum.photos/800", + h_splits: int = 2, + v_splits: int = 2, + ): + """Runs the model on the image fetched from the given url""" + image = self.fetch_image_from_url(url) + return self.get_vectors(image, h_splits, v_splits) + + def check_hard_chunking(self, quadrants): + """Check if the chunking is hard""" + variances = [] + for quadrant in quadrants: + edge_pixels = torch.cat( + [ + quadrant[0, 1], + quadrant[-1, :], + ] + ) + variances.append(torch.var(edge_pixels).item()) + return variances + + def embed_whole_image(self, image): + """Embed the entire image""" + inputs = self.processor( + image, + return_tensors="pt", + ) + with torch.no_grad(): + outputs = self.model(**inputs) + return outputs.image_embeds.squeeze() + + def apply_noise_reduction(self, image, kernel_size: int = 5): + """Implement an upscaling method to upscale the image and tiling issues""" + blur = GaussianBlur(kernel_size) + return blur(image) + + def run_from_path( + self, path: str = None, h_splits: int = 2, v_splits: int = 2 + ): + """Runs the model on the image loaded from the given path""" + image = self.load_image_from_path(path) + return self.get_vectors(image, h_splits, v_splits) + + def get_captions(self, image, candidate_captions): + """Get the best caption for the given image""" + inputs_image = self.processor( + images=image, + return_tensors="pt", + ) + + inputs_text = self.processor( + text=candidate_captions, + images=inputs_image.pixel_values[ + 0 + ], # Fix the argument name + return_tensors="pt", + padding=True, + truncation=True, + ) + + image_embeds = self.model( + pixel_values=inputs_image.pixel_values[0] + ).image_embeds + text_embeds = self.model( + input_ids=inputs_text.input_ids, + attention_mask=inputs_text.attention_mask, + ).text_embeds + + # Calculate similarity between image and text + similarities = (image_embeds @ text_embeds.T).squeeze(0) + best_caption_index = similarities.argmax().item() + + return candidate_captions[best_caption_index] + + def get_and_concat_captions( + self, image, candidate_captions, h_splits=2, v_splits=2 + ): + """Get the best caption for the given image""" + slices = self.split_image(image, h_splits, v_splits) + captions = [ + self.get_captions(slice, candidate_captions) + for slice in slices + ] + concated_captions = "".join(captions) + return concated_captions diff --git a/swarms/models/openai_models.py b/swarms/models/openai_models.py index f13657dc..b1aa0117 100644 --- a/swarms/models/openai_models.py +++ b/swarms/models/openai_models.py @@ -1,5 +1,7 @@ from __future__ import annotations +import asyncio +import functools import logging import sys from typing import ( @@ -16,6 +18,7 @@ from typing import ( Optional, Set, Tuple, + Type, Union, ) @@ -23,7 +26,7 @@ from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun, ) -from langchain.llms.base import BaseLLM, create_base_retry_decorator +from langchain.llms.base import BaseLLM from langchain.pydantic_v1 import Field, root_validator from langchain.schema import Generation, LLMResult from langchain.schema.output import GenerationChunk @@ -32,7 +35,17 @@ from langchain.utils import ( get_pydantic_field_names, ) from langchain.utils.utils import build_extra_kwargs +from tenacity import ( + RetryCallState, + before_sleep_log, + retry, + retry_base, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) +logger = logging.getLogger(__name__) from importlib.metadata import version @@ -41,6 +54,62 @@ from packaging.version import parse logger = logging.getLogger(__name__) +@functools.lru_cache +def _log_error_once(msg: str) -> None: + """Log an error once.""" + logger.error(msg) + + +def create_base_retry_decorator( + error_types: List[Type[BaseException]], + max_retries: int = 1, + run_manager: Optional[ + Union[AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun] + ] = None, +) -> Callable[[Any], Any]: + """Create a retry decorator for a given LLM and provided list of error types.""" + + _logging = before_sleep_log(logger, logging.WARNING) + + def _before_sleep(retry_state: RetryCallState) -> None: + _logging(retry_state) + if run_manager: + if isinstance(run_manager, AsyncCallbackManagerForLLMRun): + coro = run_manager.on_retry(retry_state) + try: + loop = asyncio.get_event_loop() + if loop.is_running(): + loop.create_task(coro) + else: + asyncio.run(coro) + except Exception as e: + _log_error_once(f"Error in on_retry: {e}") + else: + run_manager.on_retry(retry_state) + return None + + min_seconds = 4 + max_seconds = 10 + # Wait 2^x * 1 second between each retry starting with + # 4 seconds, then up to 10 seconds, then 10 seconds afterwards + retry_instance: "retry_base" = retry_if_exception_type( + error_types[0] + ) + for error in error_types[1:]: + retry_instance = retry_instance | retry_if_exception_type( + error + ) + return retry( + reraise=True, + stop=stop_after_attempt(max_retries), + wait=wait_exponential( + multiplier=1, min=min_seconds, max=max_seconds + ), + retry=retry_instance, + before_sleep=_before_sleep, + ) + + def is_openai_v1() -> bool: _version = parse(version("openai")) return _version.major >= 1 @@ -833,7 +902,7 @@ class OpenAIChat(BaseLLM): """ client: Any #: :meta private: - model_name: str = "gpt-3.5-turbo-1106" + model_name: str = "gpt-4-1106-preview" model_kwargs: Dict[str, Any] = Field(default_factory=dict) openai_api_key: Optional[str] = None openai_api_base: Optional[str] = None diff --git a/swarms/structs/agent.py b/swarms/structs/agent.py index dace1ff3..bd245ba7 100644 --- a/swarms/structs/agent.py +++ b/swarms/structs/agent.py @@ -9,7 +9,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple from termcolor import colored -from swarms.memory.base_vectordb import VectorDatabase +from swarms.memory.base_vectordb import AbstractDatabase from swarms.prompts.agent_system_prompts import ( AGENT_SYSTEM_PROMPT_3, ) @@ -83,7 +83,7 @@ class Agent: pdf_path (str): The path to the pdf list_of_pdf (str): The list of pdf tokenizer (Any): The tokenizer - memory (VectorDatabase): The memory + memory (AbstractDatabase): The memory preset_stopping_token (bool): Enable preset stopping token traceback (Any): The traceback traceback_handlers (Any): The traceback handlers @@ -168,7 +168,7 @@ class Agent: pdf_path: Optional[str] = None, list_of_pdf: Optional[str] = None, tokenizer: Optional[Any] = None, - long_term_memory: Optional[VectorDatabase] = None, + long_term_memory: Optional[AbstractDatabase] = None, preset_stopping_token: Optional[bool] = False, traceback: Any = None, traceback_handlers: Any = None, @@ -657,7 +657,7 @@ class Agent: """ return agent_history_prompt - def long_term_memory_prompt(self, query: str, prompt: str): + def long_term_memory_prompt(self, query: str): """ Generate the agent long term memory prompt @@ -670,10 +670,15 @@ class Agent: """ ltr = self.long_term_memory.query(query) - return f"""{prompt} - ################ CONTEXT #################### + context = f""" + {query} + ####### Long Term Memory ################ {ltr} """ + return self.short_memory.append([f"{context}"]) + + def add_memory(self, message: str): + return self.short_memory.append([f"{message}"]) async def run_concurrent(self, tasks: List[str], **kwargs): """ diff --git a/swarms/structs/conversation.py b/swarms/structs/conversation.py index 441ff3d9..a59e4cf9 100644 --- a/swarms/structs/conversation.py +++ b/swarms/structs/conversation.py @@ -60,6 +60,7 @@ class Conversation(BaseStructure): def __init__( self, + system_prompt: str, time_enabled: bool = False, database: AbstractDatabase = None, autosave: bool = False, @@ -68,12 +69,17 @@ class Conversation(BaseStructure): **kwargs, ): super().__init__() + self.system_prompt = system_prompt self.time_enabled = time_enabled self.database = database self.autosave = autosave self.save_filepath = save_filepath self.conversation_history = [] + # If system prompt is not None, add it to the conversation history + if self.system_prompt: + self.add("system", self.system_prompt) + def add(self, role: str, content: str, *args, **kwargs): """Add a message to the conversation history diff --git a/swarms/structs/multi_agent_rag.py b/swarms/structs/multi_agent_rag.py new file mode 100644 index 00000000..91d8c39d --- /dev/null +++ b/swarms/structs/multi_agent_rag.py @@ -0,0 +1,85 @@ +from dataclasses import dataclass +from typing import List, Optional + +from swarms.memory.base_vectordatabase import AbstractVectorDatabase +from swarms.structs.agent import Agent + + +@dataclass +class MultiAgentRag: + """ + Represents a multi-agent RAG (Relational Agent Graph) structure. + + Attributes: + agents (List[Agent]): List of agents in the multi-agent RAG. + db (AbstractVectorDatabase): Database used for querying. + verbose (bool): Flag indicating whether to print verbose output. + """ + + agents: List[Agent] + db: AbstractVectorDatabase + verbose: bool = False + + def query_database(self, query: str): + """ + Queries the database using the given query string. + + Args: + query (str): The query string. + + Returns: + List: The list of results from the database. + """ + results = [] + for agent in self.agents: + agent_results = agent.long_term_memory_prompt(query) + results.extend(agent_results) + return results + + def get_agent_by_id(self, agent_id) -> Optional[Agent]: + """ + Retrieves an agent from the multi-agent RAG by its ID. + + Args: + agent_id: The ID of the agent to retrieve. + + Returns: + Agent or None: The agent with the specified ID, or None if not found. + """ + for agent in self.agents: + if agent.agent_id == agent_id: + return agent + return None + + def add_message( + self, sender: Agent, message: str, *args, **kwargs + ): + """ + Adds a message to the database. + + Args: + sender (Agent): The agent sending the message. + message (str): The message to add. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + int: The ID of the added message. + """ + doc = f"{sender.ai_name}: {message}" + + return self.db.add(doc) + + def query(self, message: str, *args, **kwargs): + """ + Queries the database using the given message. + + Args: + message (str): The message to query. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + List: The list of results from the database. + """ + return self.db.query(message) diff --git a/swarms/structs/task.py b/swarms/structs/task.py index a794506f..fb89b7bf 100644 --- a/swarms/structs/task.py +++ b/swarms/structs/task.py @@ -233,6 +233,7 @@ class Task: if task.description is not None else "" ) + result = ( task.result if task.result is not None else "" ) diff --git a/swarms/telemetry/sys_info.py b/swarms/telemetry/sys_info.py index d2841585..a4857e11 100644 --- a/swarms/telemetry/sys_info.py +++ b/swarms/telemetry/sys_info.py @@ -89,7 +89,6 @@ def get_package_mismatches(file_path="pyproject.toml"): return "\n" + "\n".join(mismatches) - def system_info(): swarms_verison = get_swarms_verison() return { @@ -100,4 +99,3 @@ def system_info(): "CPU Info": get_cpu_info(), "RAM Info": get_ram_info(), } - diff --git a/swarms/telemetry/user_utils.py b/swarms/telemetry/user_utils.py index 9369bc26..4d4fb166 100644 --- a/swarms/telemetry/user_utils.py +++ b/swarms/telemetry/user_utils.py @@ -5,6 +5,7 @@ import socket from swarms.telemetry.sys_info import system_info from swarms.telemetry.check_update import check_for_package + # Helper functions def generate_user_id(): """Generate user id @@ -75,7 +76,6 @@ def get_local_ip(): return socket.gethostbyname(socket.gethostname()) - def get_user_device_data(): data = { "ID": generate_user_id(), @@ -85,5 +85,6 @@ def get_user_device_data(): "Swarms [Version]": check_for_package("swarms"), } return data - -# \ No newline at end of file + + +# diff --git a/swarms/utils/__init__.py b/swarms/utils/__init__.py index df9fe6ca..c8eaabaa 100644 --- a/swarms/utils/__init__.py +++ b/swarms/utils/__init__.py @@ -25,6 +25,14 @@ from swarms.utils.download_weights_from_url import ( from swarms.utils.save_logs import parse_log_file +######## +from swarms.utils.yaml_output_parser import YamlOutputParser +from swarms.utils.json_output_parser import JsonOutputParser +from swarms.utils.remove_json_whitespace import ( + remove_whitespace_from_json, + remove_whitespace_from_yaml, +) + __all__ = [ "SubprocessCodeInterpreter", "display_markdown_message", @@ -45,4 +53,8 @@ __all__ = [ "try_except_wrapper", "download_weights_from_url", "parse_log_file", + "YamlOutputParser", + "JsonOutputParser", + "remove_whitespace_from_json", + "remove_whitespace_from_yaml", ] diff --git a/swarms/utils/json_output_parser.py b/swarms/utils/json_output_parser.py new file mode 100644 index 00000000..724d5ed5 --- /dev/null +++ b/swarms/utils/json_output_parser.py @@ -0,0 +1,96 @@ +import json +import re +from typing import Type, TypeVar +from pydantic import BaseModel, ValidationError + +T = TypeVar("T", bound=BaseModel) + + +class JsonParsingException(Exception): + """Custom exception for errors in JSON parsing.""" + + +class JsonOutputParser: + """Parse JSON output using a Pydantic model. + + This parser is designed to extract JSON formatted data from a given string + and parse it using a specified Pydantic model for validation. + + Attributes: + pydantic_object: A Pydantic model class for parsing and validation. + pattern: A regex pattern to match JSON code blocks. + + Examples: + >>> from pydantic import BaseModel + >>> from swarms.utils.json_output_parser import JsonOutputParser + >>> class MyModel(BaseModel): + ... name: str + ... age: int + ... + >>> parser = JsonOutputParser(MyModel) + >>> text = "```json\n{\"name\": \"John\", \"age\": 42}\n```" + >>> model = parser.parse(text) + >>> model.name + + """ + + def __init__(self, pydantic_object: Type[T]): + self.pydantic_object = pydantic_object + self.pattern = re.compile( + r"^```(?:json)?(?P[^`]*)", re.MULTILINE | re.DOTALL + ) + + def parse(self, text: str) -> T: + """Parse the provided text to extract and validate JSON data. + + Args: + text: A string containing potential JSON data. + + Returns: + An instance of the specified Pydantic model with parsed data. + + Raises: + JsonParsingException: If parsing or validation fails. + """ + try: + match = re.search(self.pattern, text.strip()) + json_str = match.group("json") if match else text + + json_object = json.loads(json_str) + return self.pydantic_object.parse_obj(json_object) + + except (json.JSONDecodeError, ValidationError) as e: + name = self.pydantic_object.__name__ + msg = ( + f"Failed to parse {name} from text '{text}'." + f" Error: {e}" + ) + raise JsonParsingException(msg) from e + + def get_format_instructions(self) -> str: + """Generate formatting instructions based on the Pydantic model schema. + + Returns: + A string containing formatting instructions. + """ + schema = self.pydantic_object.schema() + reduced_schema = { + k: v + for k, v in schema.items() + if k not in ["title", "type"] + } + schema_str = json.dumps(reduced_schema, indent=4) + + format_instructions = ( + f"JSON Formatting Instructions:\n{schema_str}" + ) + return format_instructions + + +# # Example usage +# class ExampleModel(BaseModel): +# field1: int +# field2: str + +# parser = JsonOutputParser(ExampleModel) +# # Use parser.parse(text) to parse JSON data diff --git a/swarms/utils/remove_json_whitespace.py b/swarms/utils/remove_json_whitespace.py new file mode 100644 index 00000000..a5b3f7de --- /dev/null +++ b/swarms/utils/remove_json_whitespace.py @@ -0,0 +1,50 @@ +import json +import yaml + + +def remove_whitespace_from_json(json_string: str) -> str: + """ + Removes unnecessary whitespace from a JSON string. + + This function parses the JSON string into a Python object and then + serializes it back into a JSON string without unnecessary whitespace. + + Args: + json_string (str): The JSON string. + + Returns: + str: The JSON string with whitespace removed. + """ + parsed = json.loads(json_string) + return json.dumps(parsed, separators=(",", ":")) + + +# # Example usage for JSON +# json_string = '{"field1": 123, "field2": "example text"}' +# print(remove_whitespace_from_json(json_string)) + + +def remove_whitespace_from_yaml(yaml_string: str) -> str: + """ + Removes unnecessary whitespace from a YAML string. + + This function parses the YAML string into a Python object and then + serializes it back into a YAML string with minimized whitespace. + Note: This might change the representation style of YAML data. + + Args: + yaml_string (str): The YAML string. + + Returns: + str: The YAML string with whitespace reduced. + """ + parsed = yaml.safe_load(yaml_string) + return yaml.dump(parsed, default_flow_style=True) + + +# # Example usage for YAML +# yaml_string = """ +# field1: 123 +# field2: example text +# """ +# print(remove_whitespace_from_yaml(yaml_string)) diff --git a/swarms/utils/yaml_output_parser.py b/swarms/utils/yaml_output_parser.py new file mode 100644 index 00000000..61be311b --- /dev/null +++ b/swarms/utils/yaml_output_parser.py @@ -0,0 +1,89 @@ +import json +import re +import yaml +from typing import Type, TypeVar +from pydantic import BaseModel, ValidationError + +T = TypeVar("T", bound=BaseModel) + + +class YamlParsingException(Exception): + """Custom exception for errors in YAML parsing.""" + + +class YamlOutputParser: + """Parse YAML output using a Pydantic model. + + This parser is designed to extract YAML formatted data from a given string + and parse it using a specified Pydantic model for validation. + + Attributes: + pydantic_object: A Pydantic model class for parsing and validation. + pattern: A regex pattern to match YAML code blocks. + + + Examples: + >>> from pydantic import BaseModel + >>> from swarms.utils.yaml_output_parser import YamlOutputParser + >>> class MyModel(BaseModel): + ... name: str + ... age: int + ... + >>> parser = YamlOutputParser(MyModel) + >>> text = "```yaml\nname: John\nage: 42\n```" + >>> model = parser.parse(text) + >>> model.name + + """ + + def __init__(self, pydantic_object: Type[T]): + self.pydantic_object = pydantic_object + self.pattern = re.compile( + r"^```(?:ya?ml)?(?P[^`]*)", re.MULTILINE | re.DOTALL + ) + + def parse(self, text: str) -> T: + """Parse the provided text to extract and validate YAML data. + + Args: + text: A string containing potential YAML data. + + Returns: + An instance of the specified Pydantic model with parsed data. + + Raises: + YamlParsingException: If parsing or validation fails. + """ + try: + match = re.search(self.pattern, text.strip()) + yaml_str = match.group("yaml") if match else text + + json_object = yaml.safe_load(yaml_str) + return self.pydantic_object.parse_obj(json_object) + + except (yaml.YAMLError, ValidationError) as e: + name = self.pydantic_object.__name__ + msg = ( + f"Failed to parse {name} from text '{text}'." + f" Error: {e}" + ) + raise YamlParsingException(msg) from e + + def get_format_instructions(self) -> str: + """Generate formatting instructions based on the Pydantic model schema. + + Returns: + A string containing formatting instructions. + """ + schema = self.pydantic_object.schema() + reduced_schema = { + k: v + for k, v in schema.items() + if k not in ["title", "type"] + } + schema_str = json.dumps(reduced_schema, indent=4) + + format_instructions = ( + f"YAML Formatting Instructions:\n{schema_str}" + ) + return format_instructions