[DEMO][DEMOO]

.gitignore

@@ -6,6 +6,7 @@ __pycache__/
 image/
 audio/
 video/
+artifacts_three
 dataframe/
 
 static/generated
@@ -13,6 +14,8 @@ runs
 chroma
 Unit Testing Agent_state.json
 Devin_state.json
+Medical Image Diagnostic Agent_state.json
+D_state.json
 swarms/__pycache__
 artifacts
 transcript_generator.json
@@ -21,9 +24,11 @@ venv
 Cargo.lock
 .DS_STORE
 Cargo.lock
+Medical Treatment Recommendation Agent_state.json
 swarms/agents/.DS_Store
 artifacts_two
 logs
+T_state.json
 _build
 conversation.txt
 t1_state.json

new_tool_usage.py

@@ -1,68 +0,0 @@
-from pydantic import BaseModel, Field
-
-from swarms import Agent
-from swarms.models.popular_llms import Anthropic
-from swarms.tools.openai_tool_creator_decorator import tool
-
-
-# Importing the search API tool
-@tool
-def search_api(query: str) -> str:
-    """
-    This tool searches the web for information about COVID-19 symptoms.
-    """
-    return f"Search API tool called with query: {query}"
-
-
-print(search_api("COVID-19 symptoms"))
-
-
-# Initialize the schema for the person's information
-class Schema(BaseModel):
-    name: str = Field(..., title="Name of the person")
-    agent: int = Field(..., title="Age of the person")
-    is_student: bool = Field(..., title="Whether the person is a student")
-    courses: list[str] = Field(
-        ..., title="List of courses the person is taking"
-    )
-
-
-# Convert the schema to a JSON string
-tool_schema = Schema(
-    name="Tool Name",
-    agent=1,
-    is_student=True,
-    courses=["Course1", "Course2"],
-)
-
-# Define the task to generate a person's information
-task = "Generate a person's information based on the following schema:"
-
-# Initialize the agent
-agent = Agent(
-    agent_name="WeatherMan Agent",
-    # Set the tool schema to the JSON string -- this is the key difference
-    tool_schema=tool_schema,
-    llm=Anthropic(),
-    max_loops=3,
-    autosave=True,
-    dashboard=False,
-    streaming_on=True,
-    tools=[],  # or list of tools
-    verbose=True,
-    interactive=True,
-    # Set the output type to the tool schema which is a BaseModel
-    output_type=tool_schema,  # or dict, or str
-    metadata_output_type="json",
-    # List of schemas that the agent can handle
-    list_tool_schemas=[tool_schema],
-    function_calling_format_type="OpenAI",
-    function_calling_type="json",  # or soon yaml
-    execute_tool=True,
-)
-
-# Run the agent to generate the person's information
-generated_data = agent.run(task)
-
-# Print the generated data
-print(f"Generated data: {generated_data}")

playground/models/gpt4_v_example.py

@@ -1,11 +1,35 @@
-from swarms.models.gpt4v import GPT4Vision
+import os  # Import the os module for working with the operating system
 
-gpt4vision = GPT4Vision(openai_api_key="")
+from dotenv import (
+    load_dotenv,  # Import the load_dotenv function from the dotenv module
+)
 
-img = "https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/VFPt_Solenoid_correct2.svg/640px-VFPt_Solenoid_correct2.svg.png"
+from swarms import (
+    GPT4VisionAPI,  # Import the GPT4VisionAPI class from the swarms module
+)
 
+# Load the environment variables
+load_dotenv()
+
+# Get the API key from the environment variables
+api_key = os.getenv("OPENAI_API_KEY")
+
+# Create an instance of the GPT4VisionAPI class with the API key and model name
+gpt4vision = GPT4VisionAPI(
+    openai_api_key=api_key,
+    model_name="gpt-4o",
+    max_tokens=1000,
+    openai_proxy="https://api.openai.com/v1/chat/completions",
+)
+
+# Define the URL of the image to analyze
+img = "ear.png"
+
+# Define the task to perform on the image
 task = "What is this image"
 
-answer = gpt4vision.run(task, img)
+# Run the GPT4VisionAPI on the image with the specified task
+answer = gpt4vision.run(task, img, return_json=True)
 
+# Print the answer
 print(answer)

playground/models/swarms_cloud_api_example.py

@@ -0,0 +1,31 @@
+from dotenv import load_dotenv
+from openai import OpenAI
+import os
+
+load_dotenv()
+
+openai_api_key = os.getenv("SWARMS_API_KEY")
+openai_api_base = "https://api.swarms.world"
+model = "gpt-4o"
+
+client = OpenAI(api_key=openai_api_key, base_url=openai_api_base)
+# Note that this model expects the image to come before the main text
+chat_response = client.chat.completions.create(
+    model=model,
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "https://home-cdn.reolink.us/wp-content/uploads/2022/04/010345091648784709.4253.jpg",
+                    },
+                },
+                {"type": "text", "text": "What's in this image?"},
+            ],
+        }
+    ],
+    temperature=0.1,
+)
+print("Chat response:", chat_response)

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "swarms"
-version = "5.0.0"
+version = "5.0.2"
 description = "Swarms - Pytorch"
 license = "MIT"
 authors = ["Kye Gomez <kye@apac.ai>"]
@@ -48,7 +48,7 @@ Pillow = "10.3.0"
 psutil = "*"
 sentry-sdk = "*"
 python-dotenv = "*"
-accelerate = "0.28.0"
+# accelerate = "0.28.0"
 opencv-python = "^4.9.0.80"
 PyYAML = "*"
 docstring_parser = "0.16"

swarm_of_gpt4o_agents_for_octomogly.py

@@ -0,0 +1,145 @@
+import os
+
+from dotenv import load_dotenv
+from pydantic import BaseModel, Field
+
+from swarms import Agent
+from swarms.models import Anthropic
+from swarms.models.gpt4_vision_api import GPT4VisionAPI
+from swarms.structs.rearrange import AgentRearrange
+
+# Load the environment variables
+load_dotenv()
+
+
+def DIAGNOSIS_SYSTEM_PROMPT() -> str:
+    return """
+    **System Prompt for Medical Image Diagnostic Agent**
+
+    Welcome, Diagnostic Agent. Your primary function is to assist medical doctors by providing preliminary analyses of medical images. You are equipped with state-of-the-art image recognition capabilities that can identify patterns and anomalies associated with various medical conditions. Your role is to process the images, suggest potential diagnoses, and highlight areas of interest that require the doctor’s attention. 
+
+    **Guidelines:**
+
+    1. **Comprehensive Analysis:** When analyzing an image, consider all possible conditions that could explain the observed patterns. Do not jump to conclusions based on limited data. Instead, list potential diagnoses ranked by likelihood based on the image features.
+
+    2. **Explain Your Reasoning:** For each potential diagnosis, explain the specific image features and patterns that led to your conclusion. This explanation should be detailed enough for the doctor to understand your thought process and evaluate the relevance of each suggested diagnosis.
+
+    5. **Adaptability:** Be prepared to adjust your analysis based on additional information provided by the doctor or further diagnostic tests. Your ability to integrate new data and refine your assessments is crucial.
+
+    **Objective:**
+    Your goal is to enhance the diagnostic process by providing accurate, insightful, and comprehensible information that aids the doctor in making informed decisions. Remember, your analysis is a tool to support, not replace, the expertise of medical professionals.
+
+    ---
+    """
+
+
+def TREATMENT_PLAN_SYSTEM_PROMPT() -> str:
+    return """
+    **System Prompt for Medical Treatment Recommendation Agent**
+
+    Welcome, Treatment Recommendation Agent. You are tasked with assisting medical professionals by suggesting possible treatment options based on patient-specific data. Your capabilities include accessing a comprehensive database of medical treatments, understanding patient histories, and integrating the latest research to inform your recommendations.
+
+    **Guidelines:**
+
+    1. **Patient-Centric Recommendations:** Tailor your treatment suggestions to the specific needs and medical history of the patient. Consider factors such as age, pre-existing conditions, allergies, and any other relevant personal health information.
+
+    2. **Evidence-Based Suggestions:** Base your recommendations on the latest clinical guidelines and peer-reviewed research. Clearly cite the evidence supporting each suggested treatment to help the medical professional make informed decisions.
+
+    3. **Range of Options:** Provide a range of treatment possibilities that include mainstream medical practices as well as any viable alternative therapies. Classify these suggestions by effectiveness, risks, and suitability for the patient’s condition.
+
+    4. **Interdisciplinary Approach:** Encourage consideration of multidisciplinary treatment plans when appropriate. This may include combining pharmacological treatments with physical therapy, dietary changes, or psychological counseling.
+
+    5. **Clarity and Precision:** Deliver your recommendations clearly and concisely. Use plain language to describe treatment options and their implications to ensure that they are easily understood by non-specialists.
+
+    6. **Adaptability and Feedback Incorporation:** Be adaptable to feedback from the medical professionals. Use any new insights to refine future recommendations, ensuring that your system evolves in response to real-world outcomes and professional critique.
+
+    **Objective:**
+    Your primary goal is to support medical professionals by providing comprehensive, evidence-based, and personalized treatment recommendations. You serve to enhance the decision-making process, ensuring that all suggested treatments are aligned with the best interests of the patient.
+
+
+    """
+
+
+class DiagnosisSchema(BaseModel):
+    image_name: str = Field(
+        ...,
+        title="Image Name",
+        description="The name of the image to be diagnosed",
+    )
+    task: str = Field(
+        ...,
+        title="Task",
+        description="The task to be performed on the image",
+    )
+    diagnosis: str = Field(
+        ..., title="Diagnosis", description="The diagnosis of the image"
+    )
+
+
+class TreatMentSchema(BaseModel):
+    image_name: str = Field(
+        ...,
+        title="Image Name",
+        description="The name of the image to be treated",
+    )
+    task: str = Field(
+        ...,
+        title="Task",
+        description="The task to be performed on the image",
+    )
+    treatment: str = Field(
+        ..., title="Treatment", description="The treatment of the image"
+    )
+
+
+# LLM
+llm = GPT4VisionAPI(
+    openai_api_key=os.getenv("OPENAI_API_KEY"),
+    model_name="gpt-4o",
+    max_tokens=1000,
+)
+
+# Anthropic
+anthropic = Anthropic(
+    anthropic_api_key=os.getenv("ANTHROPIC_API_KEY"),
+)
+
+# Diagnoser agent
+diagnoser = Agent(
+    # agent_name="Medical Image Diagnostic Agent",
+    agent_name="D",
+    system_prompt=DIAGNOSIS_SYSTEM_PROMPT(),
+    llm=llm,
+    max_loops=1,
+    autosave=True,
+    dashboard=True,
+)
+
+
+# Agent 2 the treatment plan provider
+treatment_plan_provider = Agent(
+    # agent_name="Medical Treatment Recommendation Agent",
+    agent_name="T",
+    system_prompt=TREATMENT_PLAN_SYSTEM_PROMPT(),
+    llm=anthropic,
+    max_loops=1,
+    autosave=True,
+    dashboard=True,
+)
+
+# Agent 3 the re-arranger
+rearranger = AgentRearrange(
+    agents=[diagnoser, treatment_plan_provider],
+    flow="D -> T",
+    max_loops=1,
+    verbose=True,
+)
+
+# image = "ear.png"
+
+# Run the rearranger
+out = rearranger(
+    "Diagnose the image and provide a treatment plan for the patient",
+    # image,
+)
+print(out)

swarms/models/gpt4_vision_api.py

@@ -8,7 +8,7 @@ import aiohttp
 import requests
 from dotenv import load_dotenv
 from termcolor import colored
-
+from swarms.utils.loguru_logger import logger
 from swarms.models.base_multimodal_model import BaseMultiModalModel
 
 try:
@@ -120,7 +120,15 @@ class GPT4VisionAPI(BaseMultiModalModel):
         return base64.b64encode(response.content).decode("utf-8")
 
     # Function to handle vision tasks
-    def run(self, task: str = None, img: str = None, *args, **kwargs):
+    def run(
+        self,
+        task: str = None,
+        img: str = None,
+        multi_imgs: list = None,
+        return_json: bool = False,
+        *args,
+        **kwargs,
+    ):
         """Run the model."""
         try:
             base64_image = self.encode_image(img)
@@ -149,31 +157,27 @@ class GPT4VisionAPI(BaseMultiModalModel):
                     },
                 ],
                 "max_tokens": self.max_tokens,
+                **kwargs,
             }
             response = requests.post(
                 self.openai_proxy, headers=headers, json=payload
             )
 
-            out = response.json()
-            if "choices" in out and out["choices"]:
-                content = (
-                    out["choices"][0]
-                    .get("message", {})
-                    .get("content", None)
-                )
-                if self.streaming_enabled:
-                    content = self.stream_response(content)
-                return content
+            # Get the response as a JSON object
+            response_json = response.json()
+
+            # Return the JSON object if return_json is True
+            if return_json is True:
+                return response_json
             else:
-                print("No valid response in 'choices'")
-                return None
+                return response_json["choices"][0]["message"]["content"]
 
         except Exception as error:
-            print(
+            logger.error(
                 f"Error with the request: {error}, make sure you"
                 " double check input types and positions"
             )
-            return None
+            raise error
 
     def video_prompt(self, frames):
         """

swarms/structs/rearrange.py

@@ -1,5 +1,7 @@
-from swarms.structs.agent import Agent
 from typing import List
+
+from swarms.memory.base_vectordb import BaseVectorDatabase
+from swarms.structs.agent import Agent
 from swarms.structs.base_swarm import BaseSwarm
 from swarms.utils.loguru_logger import logger
 
@@ -27,6 +29,9 @@ class AgentRearrange(BaseSwarm):
         flow: str = None,
         max_loops: int = 1,
         verbose: bool = True,
+        memory_system: BaseVectorDatabase = None,
+        *args,
+        **kwargs,
     ):
         """
         Initializes the AgentRearrange object.
@@ -40,9 +45,15 @@ class AgentRearrange(BaseSwarm):
         self.verbose = verbose
         self.max_loops = max_loops
 
+        # Verbose is True
         if verbose is True:
             logger.add("agent_rearrange.log")
 
+        # Memory system
+        if memory_system is not None:
+            for agent in self.agents.values():
+                agent.long_term_memory = memory_system
+
     def add_agent(self, agent: Agent):
         """
         Adds an agent to the swarm.
@@ -106,7 +117,7 @@ class AgentRearrange(BaseSwarm):
         print("Flow is valid.")
         return True
 
-    def run(self, task: str, *args, **kwargs):
+    def run(self, task: str = None, img: str = None, *args, **kwargs):
         """
         Runs the swarm to rearrange the tasks.
 
@@ -123,26 +134,37 @@ class AgentRearrange(BaseSwarm):
             tasks = self.flow.split("->")
             current_task = task
 
-            for task in tasks:
-                agent_names = [name.strip() for name in task.split(",")]
-                if len(agent_names) > 1:
-                    # Parallel processing
-                    logger.info(
-                        f"Running agents in parallel: {agent_names}"
-                    )
-                    results = []
-                    for agent_name in agent_names:
-                        agent = self.agents[agent_name]
-                        result = agent.run(current_task, *args, **kwargs)
-                        results.append(result)
-                    current_task = "; ".join(results)
-                else:
-                    # Sequential processing
-                    logger.info(
-                        f"Running agents sequentially: {agent_names}"
-                    )
-                    agent = self.agents[agent_names[0]]
-                    current_task = agent.run(current_task, *args, **kwargs)
+            loop_count = 0
+            while loop_count < self.max_loops:
+                for task in tasks:
+                    agent_names = [
+                        name.strip() for name in task.split(",")
+                    ]
+                    if len(agent_names) > 1:
+                        # Parallel processing
+                        logger.info(
+                            f"Running agents in parallel: {agent_names}"
+                        )
+                        results = []
+                        for agent_name in agent_names:
+                            agent = self.agents[agent_name]
+                            result = agent.run(
+                                current_task, img, *args, **kwargs
+                            )
+                            results.append(result)
+
+                        current_task = "; ".join(results)
+                    else:
+                        # Sequential processing
+                        logger.info(
+                            f"Running agents sequentially: {agent_names}"
+                        )
+                        agent = self.agents[agent_names[0]]
+                        current_task = agent.run(
+                            current_task, img, *args, **kwargs
+                        )
+
+                loop_count += 1
 
             return current_task
         except Exception as e:
@@ -151,7 +173,11 @@ class AgentRearrange(BaseSwarm):
 
 
 def rearrange(
-    agents: List[Agent], flow: str, task: str = None, *args, **kwargs
+    agents: List[Agent] = None,
+    flow: str = None,
+    task: str = None,
+    *args,
+    **kwargs,
 ):
     """
     Rearranges the given list of agents based on the specified flow.

swarms/structs/round_robin.py

@@ -67,7 +67,9 @@ class RoundRobinSwarm(BaseSwarm):
                 for _ in range(n):
                     current_agent = self.agents[self.index]
                     try:
-                        logger.info(f"Running Agent {current_agent.agent_name} on task {result}")
+                        logger.info(
+                            f"Running Agent {current_agent.agent_name} on task {result}"
+                        )
                         result = current_agent.run(result, *args, **kwargs)
                     except Exception as e:
                         logger.error(
@@ -79,7 +81,9 @@ class RoundRobinSwarm(BaseSwarm):
                             self.index + 1
                         ) % n  # Increment and wrap around the index
                 if self.callback:
-                    logger.info(f"Calling the callback function for loop {loop}")
+                    logger.info(
+                        f"Calling the callback function for loop {loop}"
+                    )
                     self.callback(loop, result)
             return result
         except Exception as e:

tests/structs/test_round_robin_swarm.py

@@ -2,17 +2,20 @@ import pytest
 from swarms.structs.round_robin import RoundRobinSwarm
 from swarms.structs.agent import Agent
 
+
 @pytest.fixture
 def round_robin_swarm():
     agents = [Agent(name=f"Agent{i}") for i in range(3)]
     return RoundRobinSwarm(agents=agents, verbose=True, max_loops=2)
 
+
 def test_init(round_robin_swarm):
     assert isinstance(round_robin_swarm, RoundRobinSwarm)
-    assert round_robin_swarm.verbose == True
+    assert round_robin_swarm.verbose is True
     assert round_robin_swarm.max_loops == 2
     assert len(round_robin_swarm.agents) == 3
 
+
 def test_run(round_robin_swarm):
     task = "test_task"
     result = round_robin_swarm.run(task)