From f190d848a4cbe5a69946c5487ebfa772acf199ce Mon Sep 17 00:00:00 2001 From: evelynmitchell Date: Fri, 3 Nov 2023 16:13:49 -0600 Subject: [PATCH 01/32] added labeler.yml --- .github/labeler.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..72ccc40a --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,12 @@ +# this is a config file for the github action labeler + +# Add 'label1' to any changes within 'example' folder or any subfolders +example_change: +- example/** + +# Add 'label2' to any file changes within 'example2' folder +example2_change: example2/* + +# Add label3 to any change to .txt files within the entire repository. Quotation marks are required for the leading asterisk +text_files: +- '**/*.txt' \ No newline at end of file From 4fd2eab87ee0ac3c9e4970188dcc54c814ac8282 Mon Sep 17 00:00:00 2001 From: evelynmitchell Date: Fri, 3 Nov 2023 16:18:37 -0600 Subject: [PATCH 02/32] expanded permissions to allow welcome action run --- .github/workflows/welcome.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/welcome.yml b/.github/workflows/welcome.yml index eadc0b68..25edc27c 100644 --- a/.github/workflows/welcome.yml +++ b/.github/workflows/welcome.yml @@ -9,6 +9,7 @@ on: jobs: build: name: 👋 Welcome + permissions: write-all runs-on: ubuntu-latest steps: - uses: actions/first-interaction@v1.2.0 From f53236a0708e3380abbd956d11a3aba7ad1769b8 Mon Sep 17 00:00:00 2001 From: Kye Date: Fri, 3 Nov 2023 18:37:02 -0400 Subject: [PATCH 03/32] flow --- flow.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/flow.py b/flow.py index d2c21ba8..1eb46ee6 100644 --- a/flow.py +++ b/flow.py @@ -10,16 +10,10 @@ llm = OpenAIChat( max_tokens=3000, ) -# Initialize the flow +## Initialize the workflow flow = Flow( llm=llm, - max_loops=5, - dashboard=True, -) - -flow = Flow( - llm=llm, - max_loops=5, + max_loops=1, dashboard=True, # stopping_condition=None, # You can define a stopping condition as needed. # loop_interval=1, From 7d888c6a71fe8e4600458cbc03d5e649705ea30d Mon Sep 17 00:00:00 2001 From: Kye Date: Fri, 3 Nov 2023 19:57:19 -0400 Subject: [PATCH 04/32] flow example, save and load state --- flow.py | 9 +- flow_state.json | 14 ++ swarms/agents/__init__.py | 1 + swarms/models/distilled_whisperx.py | 2 +- swarms/models/huggingface.py | 2 +- swarms/structs/flow.py | 233 ++++++++++++++++++++------ swarms/structs/sequential_workflow.py | 79 +++++++++ tests/models/ada.py | 46 ++--- tests/models/huggingface.py | 23 ++- 9 files changed, 329 insertions(+), 80 deletions(-) create mode 100644 flow_state.json diff --git a/flow.py b/flow.py index 1eb46ee6..ed402a92 100644 --- a/flow.py +++ b/flow.py @@ -23,7 +23,12 @@ flow = Flow( # dynamic_temperature=False, # Set to 'True' for dynamic temperature handling. ) - +# out = flow.load_state("flow_state.json") +# temp = flow.dynamic_temperature() +# filter = flow.add_response_filter("Trump") out = flow.run("Generate a 10,000 word blog on health and wellness.") - +# out = flow.validate_response(out) +# out = flow.analyze_feedback(out) +# out = flow.print_history_and_memory() +# out = flow.save_state("flow_state.json") print(out) diff --git a/flow_state.json b/flow_state.json new file mode 100644 index 00000000..8ed134a0 --- /dev/null +++ b/flow_state.json @@ -0,0 +1,14 @@ +{ + "memory": [ + [ + "Human: Generate a 10,000 word blog on health and wellness." + ] + ], + "llm_params": {}, + "loop_interval": 1, + "retry_attempts": 3, + "retry_interval": 1, + "interactive": false, + "dashboard": true, + "dynamic_temperature": false +} \ No newline at end of file diff --git a/swarms/agents/__init__.py b/swarms/agents/__init__.py index f622f3f8..597c8c76 100644 --- a/swarms/agents/__init__.py +++ b/swarms/agents/__init__.py @@ -1,6 +1,7 @@ from swarms.agents.omni_modal_agent import OmniModalAgent from swarms.agents.hf_agents import HFAgent from swarms.agents.message import Message + # from swarms.agents.stream_response import stream from swarms.agents.base import AbstractAgent from swarms.agents.registry import Registry diff --git a/swarms/models/distilled_whisperx.py b/swarms/models/distilled_whisperx.py index 2eb2788d..8062daa4 100644 --- a/swarms/models/distilled_whisperx.py +++ b/swarms/models/distilled_whisperx.py @@ -1,3 +1,3 @@ """ -""" \ No newline at end of file +""" diff --git a/swarms/models/huggingface.py b/swarms/models/huggingface.py index 437d9144..d18b1b9d 100644 --- a/swarms/models/huggingface.py +++ b/swarms/models/huggingface.py @@ -294,7 +294,7 @@ class HuggingfaceLLM: ) print(dashboard) - + def set_device(self, device): """ Changes the device used for inference. diff --git a/swarms/structs/flow.py b/swarms/structs/flow.py index 40e00ca1..1d46678c 100644 --- a/swarms/structs/flow.py +++ b/swarms/structs/flow.py @@ -2,18 +2,16 @@ TODO: - Add tools - Add open interpreter style conversation -- Add configurable save and restore so the user can restore from previus flows - Add memory vector database retrieval """ import json import logging import time -from typing import Any, Callable, Dict, List, Optional, Tuple, Generator +from typing import Any, Callable, Dict, List, Optional, Tuple from termcolor import colored import inspect import random -# from swarms.tools.tool import BaseTool # Constants @@ -36,7 +34,6 @@ When you have finished the task, and you feel as if you are done: output a speci This will enable you to leave the flow loop. """ - # Custome stopping condition def stop_when_repeats(response: str) -> bool: # Stop if the word stop appears in the response @@ -209,7 +206,7 @@ class Flow: print(dashboard) - def run(self, task: str, **kwargs): + def run(self, task: str, save: bool = True, **kwargs): """ Run the autonomous agent loop @@ -223,7 +220,16 @@ class Flow: 4. If stopping condition is not met, generate a response 5. Repeat until stopping condition is met or max_loops is reached + Example: + >>> out = flow.run("Generate a 10,000 word blog on health and wellness.") + """ + # Start with a new history or continue from the last saved state + if not self.memory or not self.memory[-1]: + history = [f"Human: {task}"] + else: + history = self.memory[-1] + response = task history = [f"Human: {task}"] @@ -231,9 +237,12 @@ class Flow: if self.dashboard: self.print_dashboard(task) - for i in range(self.max_loops): + # Start or continue the loop process + for i in range(len(history), self.max_loops): print(colored(f"\nLoop {i+1} of {self.max_loops}", "blue")) print("\n") + response = history[-1].split(": ", 1)[-1] # Get the last response + if self._check_stopping_condition(response) or parse_done_token(response): break @@ -245,15 +254,8 @@ class Flow: while attempt < self.retry_attempts: try: response = self.llm( - f""" - SYSTEM_PROMPT: - {FLOW_SYSTEM_PROMPT} - - - History: {response} - - """, - **kwargs, + self.agent_history_prompt(FLOW_SYSTEM_PROMPT, response) + ** kwargs, ) # print(f"Next query: {response}") # break @@ -274,6 +276,10 @@ class Flow: history.append(response) time.sleep(self.loop_interval) self.memory.append(history) + + if save: + self.save("flow_history.json") + return response # , history def _run(self, **kwargs: Any) -> str: @@ -283,32 +289,31 @@ class Flow: logging.info(f"Message history: {history}") return response - def bulk_run(self, inputs: List[Dict[str, Any]]) -> List[str]: - """Generate responses for multiple input sets.""" - return [self.run(**input_data) for input_data in inputs] - - def run_dynamically(self, task: str, max_loops: Optional[int] = None): + def agent_history_prompt( + self, + system_prompt: str = FLOW_SYSTEM_PROMPT, + history=None, + ): """ - Run the autonomous agent loop dynamically based on the - - # Usage Example + Generate the agent history prompt - # Initialize the Flow - flow = Flow(llm=lambda x: x, max_loops=5) - - # Run dynamically based on token and optional max loops - response = flow.run_dynamically("Generate a report ", max_loops=3) - print(response) + Args: + system_prompt (str): The system prompt + history (List[str]): The history of the conversation - response = flow.run_dynamically("Generate a report ") - print(response) + Returns: + str: The agent history prompt + """ + agent_history_prompt = f""" + SYSTEM_PROMPT: {system_prompt} + History: {history} """ - if "" in task: - self.stopping_condition = parse_done_token - self.max_loops = max_loops or float("inf") - response = self.run(task) - return response + return agent_history_prompt + + def bulk_run(self, inputs: List[Dict[str, Any]]) -> List[str]: + """Generate responses for multiple input sets.""" + return [self.run(**input_data) for input_data in inputs] @staticmethod def from_llm_and_template(llm: Any, template: str) -> "Flow": @@ -339,6 +344,60 @@ class Flow: return False return True + def print_history_and_memory(self): + """ + Prints the entire history and memory of the flow. + Each message is colored and formatted for better readability. + """ + print(colored("Flow History and Memory", "cyan", attrs=["bold"])) + print(colored("========================", "cyan", attrs=["bold"])) + for loop_index, history in enumerate(self.memory, start=1): + print(colored(f"\nLoop {loop_index}:", "yellow", attrs=["bold"])) + for message in history: + speaker, _, message_text = message.partition(": ") + if "Human" in speaker: + print(colored(f"{speaker}:", "green") + f" {message_text}") + else: + print(colored(f"{speaker}:", "blue") + f" {message_text}") + print(colored("------------------------", "cyan")) + print(colored("End of Flow History", "cyan", attrs=["bold"])) + + def step(self, task: str, **kwargs): + """ + + Executes a single step in the flow interaction, generating a response + from the language model based on the given input text. + + Args: + input_text (str): The input text to prompt the language model with. + + Returns: + str: The language model's generated response. + + Raises: + Exception: If an error occurs during response generation. + + """ + try: + # Generate the response using lm + response = self.llm(task, **kwargs) + + # Update the flow's history with the new interaction + if self.interactive: + self.memory.append(f"AI: {response}") + self.memory.append(f"Human: {task}") + else: + self.memory.append(f"AI: {response}") + + return response + except Exception as error: + logging.error(f"Error generating response: {error}") + raise + + def graceful_shutdown(self): + """Gracefully shutdown the system saving the state""" + return self.save_state("flow_state.json") + def run_with_timeout(self, task: str, timeout: int = 60) -> str: """Run the loop but stop if it takes longer than the timeout""" start_time = time.time() @@ -455,23 +514,97 @@ class Flow: print() return response - def streamed_token_generation(self, prompt: str) -> Generator[str, None, None]: + def get_llm_params(self): + """ + Extracts and returns the parameters of the llm object for serialization. + It assumes that the llm object has an __init__ method with parameters that can be used to recreate it. """ - Generate tokens in real-time for a given prompt. + if not hasattr(self.llm, "__init__"): + return None - This method simulates the real-time generation of each token. - For simplicity, we treat each character of the input as a token - and yield them with a slight delay. In a real-world scenario, - this would involve using the LLM's internal methods to generate - the response token by token. + init_signature = inspect.signature(self.llm.__init__) + params = init_signature.parameters + llm_params = {} + + for name, param in params.items(): + if name == "self": + continue + if hasattr(self.llm, name): + value = getattr(self.llm, name) + if isinstance( + value, (str, int, float, bool, list, dict, tuple, type(None)) + ): + llm_params[name] = value + else: + llm_params[name] = str( + value + ) # For non-serializable objects, save their string representation. + + return llm_params + + def save_state(self, file_path: str) -> None: + """ + Saves the current state of the flow to a JSON file, including the llm parameters. Args: - prompt (str): The input prompt for which the tokens should be generated. + file_path (str): The path to the JSON file where the state will be saved. - Yields: - str: The next token (character) from the generated response. + Example: + >>> flow.save_state('saved_flow.json') """ - tokens = list(prompt) - for token in tokens: - time.sleep(0.1) - yield token + state = { + "memory": self.memory, + # "llm_params": self.get_llm_params(), + "loop_interval": self.loop_interval, + "retry_attempts": self.retry_attempts, + "retry_interval": self.retry_interval, + "interactive": self.interactive, + "dashboard": self.dashboard, + "dynamic_temperature": self.dynamic_temperature, + } + + with open(file_path, "w") as f: + json.dump(state, f, indent=4) + + saved = colored("Saved flow state to", "green") + print(f"{saved} {file_path}") + + def load_state(self, file_path: str): + """ + Loads the state of the flow from a json file and restores the configuration and memory. + + + Example: + >>> flow = Flow(llm=llm_instance, max_loops=5) + >>> flow.load_state('saved_flow.json') + >>> flow.run("Continue with the task") + + """ + with open(file_path, "r") as f: + state = json.load(f) + + # Assuming 'llm_class' is a class reference to the language + # llm_params = state.get("llm_params", {}) + # self.llm = self.llm(**llm_params) + + # Restore other saved attributes + self.memory = state.get("memory", []) + self.max_loops = state.get("max_loops", 5) + self.loop_interval = state.get("loop_interval", 1) + self.retry_attempts = state.get("retry_attempts", 3) + self.retry_interval = state.get("retry_interval", 1) + self.interactive = state.get("interactive", False) + + print(f"Flow state loaded from {file_path}") + + def retry_on_failure(self, function, retries: int = 3, retry_delay: int = 1): + """Retry wrapper for LLM calls.""" + attempt = 0 + while attempt < retries: + try: + return function() + except Exception as error: + logging.error(f"Error generating response: {error}") + attempt += 1 + time.sleep(retry_delay) + raise Exception("All retry attempts failed") diff --git a/swarms/structs/sequential_workflow.py b/swarms/structs/sequential_workflow.py index 2df95c07..f27f3989 100644 --- a/swarms/structs/sequential_workflow.py +++ b/swarms/structs/sequential_workflow.py @@ -18,3 +18,82 @@ workflow.add("Create a report on these metrics", mistral) workflow.run() """ +from dataclasses import dataclass, field +from typing import List, Any, Dict, Callable, Union +from swarms.models import OpenAIChat +from swarms.structs import Flow + + +# Define a generic Task that can handle different types of callable objects +@dataclass +class Task: + description: str + model: Union[Callable, Flow] + args: List[Any] = field(default_factory=list) + kwargs: Dict[str, Any] = field(default_factory=dict) + result: Any = None + + def execute(self): + if isinstance(self.model, Flow): + self.result = self.model.run(*self.args, **self.kwargs) + else: + self.result = self.model(*self.args, **self.kwargs) + + +# SequentialWorkflow class definition using dataclasses +@dataclass +class SequentialWorkflow: + tasks: List[Task] = field(default_factory=list) + max_loops: int = 1 + + def add( + self, description: str, model: Union[Callable, Flow], *args, **kwargs + ) -> None: + self.tasks.append( + Task(description=description, model=model, args=list(args), kwargs=kwargs) + ) + + def run(self) -> None: + for _ in range(self.max_loops): + for task in self.tasks: + # Check if the current task can be executed + if task.result is None: + task.execute() + # Pass the result as an argument to the next task if it exists + next_task_index = self.tasks.index(task) + 1 + if next_task_index < len(self.tasks): + next_task = self.tasks[next_task_index] + next_task.args.insert(0, task.result) + + +# Example usage +api_key = "" # Your actual API key here + +# Initialize the language model +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) + +# Initialize the Flow with the language model +flow1 = Flow(llm=llm, max_loops=5, dashboard=True) + +# Create another Flow for a different task +flow2 = Flow(llm=llm, max_loops=5, dashboard=True) + +# Create the workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) + +# Suppose the next task takes the output of the first task as input +workflow.add("Summarize the generated blog", flow2) + +# Run the workflow +workflow.run() + +# Output the results +for task in workflow.tasks: + print(f"Task: {task.description}, Result: {task.result}") diff --git a/tests/models/ada.py b/tests/models/ada.py index 786b162d..08f1a687 100644 --- a/tests/models/ada.py +++ b/tests/models/ada.py @@ -3,12 +3,15 @@ import pytest import openai from unittest.mock import patch -from swarms.models.simple_ada import get_ada_embeddings # Adjust this import path to your project structure +from swarms.models.simple_ada import ( + get_ada_embeddings, +) # Adjust this import path to your project structure from os import getenv from dotenv import load_dotenv load_dotenv() + # Fixture for test texts @pytest.fixture def test_texts(): @@ -18,20 +21,24 @@ def test_texts(): "A quick brown fox jumps over the lazy dog", ] + # Basic Test def test_get_ada_embeddings_basic(test_texts): - with patch('openai.Embedding.create') as mock_create: + with patch("openai.Embedding.create") as mock_create: # Mocking the OpenAI API call - mock_create.return_value = { - "data": [ - {"embedding": [0.1, 0.2, 0.3]} - ] - } - + mock_create.return_value = {"data": [{"embedding": [0.1, 0.2, 0.3]}]} + for text in test_texts: embedding = get_ada_embeddings(text) - assert embedding == [0.1, 0.2, 0.3], "Embedding does not match expected output" - mock_create.assert_called_with(input=[text.replace("\n", " ")], model="text-embedding-ada-002") + assert embedding == [ + 0.1, + 0.2, + 0.3, + ], "Embedding does not match expected output" + mock_create.assert_called_with( + input=[text.replace("\n", " ")], model="text-embedding-ada-002" + ) + # Parameterized Test @pytest.mark.parametrize( @@ -42,27 +49,28 @@ def test_get_ada_embeddings_basic(test_texts): ], ) def test_get_ada_embeddings_models(text, model, expected_call_model): - with patch('openai.Embedding.create') as mock_create: - mock_create.return_value = { - "data": [ - {"embedding": [0.1, 0.2, 0.3]} - ] - } + with patch("openai.Embedding.create") as mock_create: + mock_create.return_value = {"data": [{"embedding": [0.1, 0.2, 0.3]}]} _ = get_ada_embeddings(text, model=model) mock_create.assert_called_with(input=[text], model=expected_call_model) + # Exception Test def test_get_ada_embeddings_exception(): - with patch('openai.Embedding.create') as mock_create: + with patch("openai.Embedding.create") as mock_create: mock_create.side_effect = openai.error.OpenAIError("Test error") with pytest.raises(openai.error.OpenAIError): get_ada_embeddings("Some text") + # Tests for environment variable loading def test_env_var_loading(monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "testkey123") - with patch('openai.Embedding.create'): - assert getenv("OPENAI_API_KEY") == "testkey123", "Environment variable for API key is not set correctly" + with patch("openai.Embedding.create"): + assert ( + getenv("OPENAI_API_KEY") == "testkey123" + ), "Environment variable for API key is not set correctly" + # ... more tests to cover other aspects such as different input types, large inputs, invalid inputs, etc. diff --git a/tests/models/huggingface.py b/tests/models/huggingface.py index 847ced06..71fefa67 100644 --- a/tests/models/huggingface.py +++ b/tests/models/huggingface.py @@ -70,11 +70,14 @@ def test_llm_memory_consumption(llm_instance): # Test different initialization parameters -@pytest.mark.parametrize("model_id, max_length", [ - ("gpt2-small", 100), - ("gpt2-medium", 200), - ("gpt2-large", None) # None to check default behavior -]) +@pytest.mark.parametrize( + "model_id, max_length", + [ + ("gpt2-small", 100), + ("gpt2-medium", 200), + ("gpt2-large", None), # None to check default behavior + ], +) def test_llm_initialization_params(model_id, max_length): if max_length: instance = HuggingfaceLLM(model_id=model_id, max_length=max_length) @@ -157,11 +160,14 @@ def test_llm_timeout_handling(mock_run, llm_instance): @patch("swarms.models.huggingface.HuggingfaceLLM.run") def test_llm_response_time(mock_run, llm_instance): import time + mock_run.return_value = "mocked output" start_time = time.time() llm_instance.run("test task for response time") end_time = time.time() - assert end_time - start_time < 1 # Assuming the response should be faster than 1 second + assert ( + end_time - start_time < 1 + ) # Assuming the response should be faster than 1 second # Test the logging of a warning for long inputs @@ -173,7 +179,9 @@ def test_llm_long_input_warning(mock_warning, llm_instance): # Test for run method behavior when model raises an exception -@patch("swarms.models.huggingface.HuggingfaceLLM._model.generate", side_effect=RuntimeError) +@patch( + "swarms.models.huggingface.HuggingfaceLLM._model.generate", side_effect=RuntimeError +) def test_llm_run_model_exception(mock_generate, llm_instance): with pytest.raises(RuntimeError): llm_instance.run("test task when model fails") @@ -219,6 +227,7 @@ def test_llm_multilingual_input(mock_run, llm_instance): result = llm_instance.run(multilingual_input) assert isinstance(result, str) # Simple check to ensure output is string type + # Test caching mechanism to prevent re-running the same inputs @patch("swarms.models.huggingface.HuggingfaceLLM.run") def test_llm_caching_mechanism(mock_run, llm_instance): From 154f50cc25eba4fc55866f5f7ae715acb3417f85 Mon Sep 17 00:00:00 2001 From: Kye Date: Sat, 4 Nov 2023 15:58:32 -0400 Subject: [PATCH 05/32] groupchat --- groupchat.py | 136 ++++++++------------------- swarms/structs/flow.py | 106 ++++++++++++++++++++- swarms/swarms/groupchat.py | 185 ++++++++++++++++++++----------------- 3 files changed, 241 insertions(+), 186 deletions(-) diff --git a/groupchat.py b/groupchat.py index 6694d71f..a97fbdd4 100644 --- a/groupchat.py +++ b/groupchat.py @@ -1,109 +1,49 @@ -# from swarms.structs import Flow -# from swarms.models import OpenAIChat -# from swarms.swarms.groupchat import GroupChat -# from swarms.agents import SimpleAgent +from swarms import OpenAI, Flow +from swarms.swarms.groupchat import GroupChatManager, GroupChat -# api_key = "" -# llm = OpenAIChat( -# openai_api_key=api_key, -# ) +api_key = "" -# agent1 = SimpleAgent("Captain Price", Flow(llm=llm, max_loops=4)) -# agent2 = SimpleAgent("John Mactavis", Flow(llm=llm, max_loops=4)) - -# # Create a groupchat with the 2 agents -# chat = GroupChat([agent1, agent2]) - -# # Assign duties to the agents -# chat.assign_duty(agent1.name, "Buy the groceries") -# chat.assign_duty(agent2.name, "Clean the house") - -# # Initate a chat -# response = chat.run("Captain Price", "Hello, how are you John?") -# print(response) - - -from swarms.models import OpenAIChat -from swarms.structs import Flow -import random - -api_key = "" # Your API Key here - - -class GroupChat: - """ - GroupChat class that facilitates agent-to-agent communication using multiple instances of the Flow class. - """ - - def __init__(self, agents: list): - self.agents = {f"agent_{i}": agent for i, agent in enumerate(agents)} - self.message_log = [] - - def add_agent(self, agent: Flow): - agent_id = f"agent_{len(self.agents)}" - self.agents[agent_id] = agent - - def remove_agent(self, agent_id: str): - if agent_id in self.agents: - del self.agents[agent_id] - - def send_message(self, sender_id: str, recipient_id: str, message: str): - if sender_id not in self.agents or recipient_id not in self.agents: - raise ValueError("Invalid sender or recipient ID.") - formatted_message = f"{sender_id} to {recipient_id}: {message}" - self.message_log.append(formatted_message) - recipient_agent = self.agents[recipient_id] - recipient_agent.run(message) - - def broadcast_message(self, sender_id: str, message: str): - for agent_id, agent in self.agents.items(): - if agent_id != sender_id: - self.send_message(sender_id, agent_id, message) - - def get_message_log(self): - return self.message_log - - -class EnhancedGroupChatV2(GroupChat): - def __init__(self, agents: list): - super().__init__(agents) - - def multi_round_conversation(self, rounds: int = 5): - """ - Initiate a multi-round conversation between agents. - - Args: - rounds (int): The number of rounds of conversation. - """ - for _ in range(rounds): - # Randomly select a sender and recipient agent for the conversation - sender_id = random.choice(list(self.agents.keys())) - recipient_id = random.choice(list(self.agents.keys())) - while recipient_id == sender_id: # Ensure the recipient is not the sender - recipient_id = random.choice(list(self.agents.keys())) - - # Generate a message (for simplicity, a generic message is used) - message = f"Hello {recipient_id}, how are you today?" - self.send_message(sender_id, recipient_id, message) - - -# Sample usage with EnhancedGroupChatV2 -# Initialize the language model -llm = OpenAIChat( +llm = OpenAI( openai_api_key=api_key, temperature=0.5, max_tokens=3000, ) -# Initialize two Flow agents -agent1 = Flow(llm=llm, max_loops=5, dashboard=True) -agent2 = Flow(llm=llm, max_loops=5, dashboard=True) +# Initialize the flow +flow1 = Flow( + llm=llm, + max_loops=1, + system_message="YOU ARE SILLY, YOU OFFER NOTHING OF VALUE", + name='silly', + dashboard=True, +) +flow2 = Flow( + llm=llm, + max_loops=1, + system_message="YOU ARE VERY SMART AND ANSWER RIDDLES", + name='detective', + dashboard=True, +) +flow3 = Flow( + llm=llm, + max_loops=1, + system_message="YOU MAKE RIDDLES", + name='riddler', + dashboard=True, +) +manager = Flow( + llm=llm, + max_loops=1, + system_message="YOU ARE A GROUP CHAT MANAGER", + name='manager', + dashboard=True, +) -# Create an enhanced group chat with the two agents -enhanced_group_chat_v2 = EnhancedGroupChatV2(agents=[agent1, agent2]) -# Simulate multi-round agent to agent communication -enhanced_group_chat_v2.multi_round_conversation(rounds=5) +# Example usage: +agents = [flow1, flow2, flow3] -enhanced_group_chat_v2.get_message_log() # Get the conversation log +group_chat = GroupChat(agents=agents, messages=[], max_round=10) +chat_manager = GroupChatManager(groupchat=group_chat, selector = manager) +chat_history = chat_manager("Write me a riddle") \ No newline at end of file diff --git a/swarms/structs/flow.py b/swarms/structs/flow.py index 1d46678c..afbcf536 100644 --- a/swarms/structs/flow.py +++ b/swarms/structs/flow.py @@ -34,6 +34,7 @@ When you have finished the task, and you feel as if you are done: output a speci This will enable you to leave the flow loop. """ + # Custome stopping condition def stop_when_repeats(response: str) -> bool: # Stop if the word stop appears in the response @@ -100,6 +101,8 @@ class Flow: retry_interval: int = 1, interactive: bool = False, dashboard: bool = False, + name: str = "Flow agent", + system_message: str = FLOW_SYSTEM_PROMPT, # tools: List[BaseTool] = None, dynamic_temperature: bool = False, **kwargs: Any, @@ -119,6 +122,8 @@ class Flow: self.dashboard = dashboard self.dynamic_temperature = dynamic_temperature # self.tools = tools + self.system_message = system_message + self.name = name def provide_feedback(self, feedback: str) -> None: """Allow users to provide feedback on the responses.""" @@ -131,11 +136,6 @@ class Flow: return self.stopping_condition(response) return False - def __call__(self, prompt, **kwargs) -> str: - """Invoke the flow by providing a template and its variables.""" - response = self.llm(prompt, **kwargs) - return response - def dynamic_temperature(self): """ 1. Check the self.llm object for the temperature @@ -282,6 +282,82 @@ class Flow: return response # , history + def __call__(self, task: str, save: bool = True, **kwargs): + """ + Run the autonomous agent loop + + Args: + task (str): The initial task to run + + Flow: + 1. Generate a response + 2. Check stopping condition + 3. If stopping condition is met, stop + 4. If stopping condition is not met, generate a response + 5. Repeat until stopping condition is met or max_loops is reached + + Example: + >>> out = flow.run("Generate a 10,000 word blog on health and wellness.") + + """ + # Start with a new history or continue from the last saved state + if not self.memory or not self.memory[-1]: + history = [f"Human: {task}"] + else: + history = self.memory[-1] + + response = task + history = [f"Human: {task}"] + + # If dashboard = True then print the dashboard + if self.dashboard: + self.print_dashboard(task) + + # Start or continue the loop process + for i in range(len(history), self.max_loops): + print(colored(f"\nLoop {i+1} of {self.max_loops}", "blue")) + print("\n") + response = history[-1].split(": ", 1)[-1] # Get the last response + + if self._check_stopping_condition(response) or parse_done_token(response): + break + + # Adjust temperature, comment if no work + if self.dynamic_temperature: + self.dynamic_temperature() + + attempt = 0 + while attempt < self.retry_attempts: + try: + response = self.llm( + self.agent_history_prompt(FLOW_SYSTEM_PROMPT, response) + ** kwargs, + ) + # print(f"Next query: {response}") + # break + if self.interactive: + print(f"AI: {response}") + history.append(f"AI: {response}") + response = input("You: ") + history.append(f"Human: {response}") + else: + print(f"AI: {response}") + history.append(f"AI: {response}") + print(response) + break + except Exception as e: + logging.error(f"Error generating response: {e}") + attempt += 1 + time.sleep(self.retry_interval) + history.append(response) + time.sleep(self.loop_interval) + self.memory.append(history) + + if save: + self.save_state("flow_history.json") + + return response # , history + def _run(self, **kwargs: Any) -> str: """Generate a result using the provided keyword args.""" task = self.format_prompt(**kwargs) @@ -304,6 +380,7 @@ class Flow: Returns: str: The agent history prompt """ + system_prompt = system_prompt or self.system_message agent_history_prompt = f""" SYSTEM_PROMPT: {system_prompt} @@ -608,3 +685,22 @@ class Flow: attempt += 1 time.sleep(retry_delay) raise Exception("All retry attempts failed") + + def generate_reply(self, history: str, **kwargs) -> str: + """ + Generate a response based on initial or task + """ + prompt = f""" + + SYSTEM_PROMPT: {self.system_message} + + History: {history} + + Your response: + """ + response = self.llm(prompt, **kwargs) + return {"role": self.name, "content": response} + + def update_system_message(self, system_message: str): + """Upddate the system message""" + self.system_message = system_message diff --git a/swarms/swarms/groupchat.py b/swarms/swarms/groupchat.py index 6f5f43b6..6bbe0898 100644 --- a/swarms/swarms/groupchat.py +++ b/swarms/swarms/groupchat.py @@ -1,89 +1,108 @@ -from swarms.agents import SimpleAgent -from termcolor import colored +import logging +from dataclasses import dataclass +from typing import Dict, List +from swarms.structs.flow import Flow +logger = logging.getLogger(__name__) + + +@dataclass class GroupChat: - """ - Groupchat - - Args: - agents (list): List of agents - dashboard (bool): Whether to print a dashboard or not - - Example: - >>> from swarms.structs import Flow - >>> from swarms.models import OpenAIChat - >>> from swarms.swarms.groupchat import GroupChat - >>> from swarms.agents import SimpleAgent - >>> api_key = "" - >>> llm = OpenAIChat() - >>> agent1 = SimpleAgent("Captain Price", Flow(llm=llm, max_loops=4)) - >>> agent2 = SimpleAgent("John Mactavis", Flow(llm=llm, max_loops=4)) - >>> chat = GroupChat([agent1, agent2]) - >>> chat.assign_duty(agent1.name, "Buy the groceries") - >>> chat.assign_duty(agent2.name, "Clean the house") - >>> response = chat.run("Captain Price", "Hello, how are you John?") - >>> print(response) - - - - """ - - def __init__(self, agents, dashboard: bool = False): - # Ensure that all provided agents are instances of simpleagents - if not all(isinstance(agent, SimpleAgent) for agent in agents): - raise ValueError("All agents must be instances of SimpleAgent") - self.agents = {agent.name: agent for agent in agents} - - # Dictionary to store duties for each agent - self.duties = {} - - # Dictionary to store roles for each agent - self.roles = {} - - self.dashboard = dashboard - - def assign_duty(self, agent_name, duty): - """Assigns duty to the agent""" - if agent_name not in self.agents: - raise ValueError(f"No agent named {agent_name} found.") - - def assign_role(self, agent_name, role): - """Assigns a role to the specified agent""" - if agent_name not in self.agents: - raise ValueError(f"No agent named {agent_name} found") - - self.roles[agent_name] = role - - def run(self, sender_name: str, message: str): - """Runs the groupchat""" - if self.dashboard: - metrics = print( - colored( - f""" - - Groupchat Configuration: - ------------------------ - - Agents: {self.agents} - Message: {message} - Sender: {sender_name} - """, - "red", - ) + """A group chat class that contains a list of agents and the maximum number of rounds.""" + + agents: List[Flow] + messages: List[Dict] + max_round: int = 10 + admin_name: str = "Admin" # the name of the admin agent + + @property + def agent_names(self) -> List[str]: + """Return the names of the agents in the group chat.""" + return [agent.name for agent in self.agents] + + def reset(self): + """Reset the group chat.""" + self.messages.clear() + + def agent_by_name(self, name: str) -> Flow: + """Find an agent whose name is contained within the given 'name' string.""" + for agent in self.agents: + if agent.name in name: + return agent + raise ValueError(f"No agent found with a name contained in '{name}'.") + + def next_agent(self, agent: Flow) -> Flow: + """Return the next agent in the list.""" + return self.agents[(self.agent_names.index(agent.name) + 1) % len(self.agents)] + + def select_speaker_msg(self): + """Return the message for selecting the next speaker.""" + return f""" + You are in a role play game. The following roles are available: + {self._participant_roles()}. + + Read the following conversation. + Then select the next role from {self.agent_names} to play. Only return the role. + """ + + def select_speaker(self, last_speaker: Flow, selector: Flow): + """Select the next speaker.""" + selector.update_system_message(self.select_speaker_msg()) + + # Warn if GroupChat is underpopulated, without established changing behavior + n_agents = len(self.agent_names) + if n_agents < 3: + logger.warning( + f"GroupChat is underpopulated with {n_agents} agents. Direct communication would be more efficient." ) - print(metrics) - - responses = {} - for agent_name, agent in self.agents.items(): - if agent_name != sender_name: - if agent_name in self.duties: - message += f"Your duty is {self.duties[agent_name]}" - if agent_name in self.roles: - message += ( - f"You are the {self.roles[agent_name]} in this conversation" - ) + name = selector.generate_reply( + self.format_history( + self.messages + + [ + { + "role": "system", + "content": f"Read the above conversation. Then select the next most suitable role from {self.agent_names} to play. Only return the role.", + } + ] + ) + ) + try: + return self.agent_by_name(name["content"]) + except ValueError: + return self.next_agent(last_speaker) + + def _participant_roles(self): + return "\n".join( + [f"{agent.name}: {agent.system_message}" for agent in self.agents] + ) + + def format_history(self, messages: List[Dict]) -> str: + formatted_messages = [] + for message in messages: + formatted_message = f"'{message['role']}:{message['content']}" + formatted_messages.append(formatted_message) + return "\n".join(formatted_messages) + + +class GroupChatManager: + def __init__(self, groupchat: GroupChat, selector: Flow): + self.groupchat = groupchat + self.selector = selector + + def __call__(self, task: str): + self.groupchat.messages.append({"role": self.selector.name, "content": task}) + for i in range(self.groupchat.max_round): + speaker = self.groupchat.select_speaker( + last_speaker=self.selector, selector=self.selector + ) + reply = speaker.generate_reply( + self.groupchat.format_history(self.groupchat.messages) + ) + self.groupchat.messages.append(reply) + print(reply) + if i == self.groupchat.max_round - 1: + break - responses[agent_name] = agent.run(message) - return responses + return reply From 2f31a6349419f122ad36e47516e185ca19bbdc6d Mon Sep 17 00:00:00 2001 From: Kye Date: Sat, 4 Nov 2023 16:20:51 -0400 Subject: [PATCH 06/32] flow -> example.py --- example.py | 35 +++++++++++++++++++++++------------ flow.py | 34 ---------------------------------- 2 files changed, 23 insertions(+), 46 deletions(-) delete mode 100644 flow.py diff --git a/example.py b/example.py index e9dfac18..eb750eb7 100644 --- a/example.py +++ b/example.py @@ -1,24 +1,35 @@ from swarms.models import OpenAIChat -from swarms import Worker -from swarms.prompts import PRODUCT_AGENT_PROMPT +from swarms.structs import Flow api_key = "" +# Initialize the language model, this model can be swapped out with Anthropic, ETC, Huggingface Models like Mistral, ETC llm = OpenAIChat( + # model_name="gpt-4" openai_api_key=api_key, temperature=0.5, + #max_tokens=100, ) -node = Worker( +## Initialize the workflow +flow = Flow( llm=llm, - ai_name="Optimus Prime", - openai_api_key=api_key, - ai_role=PRODUCT_AGENT_PROMPT, - external_tools=None, - human_in_the_loop=False, - temperature=0.5, + max_loops=1, + dashboard=True, + # stopping_condition=None, # You can define a stopping condition as needed. + # loop_interval=1, + # retry_attempts=3, + # retry_interval=1, + # interactive=False, # Set to 'True' for interactive mode. + # dynamic_temperature=False, # Set to 'True' for dynamic temperature handling. ) -task = "Locate 5 trending topics on healthy living, locate a website like NYTimes, and then generate an image of people doing those topics." -response = node.run(task) -print(response) +# out = flow.load_state("flow_state.json") +# temp = flow.dynamic_temperature() +# filter = flow.add_response_filter("Trump") +out = flow.run("Generate a 10,000 word blog on health and wellness.") +# out = flow.validate_response(out) +# out = flow.analyze_feedback(out) +# out = flow.print_history_and_memory() +# out = flow.save_state("flow_state.json") +print(out) diff --git a/flow.py b/flow.py deleted file mode 100644 index ed402a92..00000000 --- a/flow.py +++ /dev/null @@ -1,34 +0,0 @@ -from swarms.models import OpenAIChat -from swarms.structs import Flow - -api_key = "" - -# Initialize the language model, this model can be swapped out with Anthropic, ETC, Huggingface Models like Mistral, ETC -llm = OpenAIChat( - openai_api_key=api_key, - temperature=0.5, - max_tokens=3000, -) - -## Initialize the workflow -flow = Flow( - llm=llm, - max_loops=1, - dashboard=True, - # stopping_condition=None, # You can define a stopping condition as needed. - # loop_interval=1, - # retry_attempts=3, - # retry_interval=1, - # interactive=False, # Set to 'True' for interactive mode. - # dynamic_temperature=False, # Set to 'True' for dynamic temperature handling. -) - -# out = flow.load_state("flow_state.json") -# temp = flow.dynamic_temperature() -# filter = flow.add_response_filter("Trump") -out = flow.run("Generate a 10,000 word blog on health and wellness.") -# out = flow.validate_response(out) -# out = flow.analyze_feedback(out) -# out = flow.print_history_and_memory() -# out = flow.save_state("flow_state.json") -print(out) From 75ebbe04f8ceabb85149afac9a177c25ce699dcc Mon Sep 17 00:00:00 2001 From: Kye Date: Sat, 4 Nov 2023 16:53:48 -0400 Subject: [PATCH 07/32] distilled whisperx --- example.py | 2 +- groupchat.py | 12 +-- swarms/models/__init__.py | 1 + swarms/models/distilled_whisperx.py | 161 +++++++++++++++++++++++++++- 4 files changed, 167 insertions(+), 9 deletions(-) diff --git a/example.py b/example.py index eb750eb7..aeae1b02 100644 --- a/example.py +++ b/example.py @@ -8,7 +8,7 @@ llm = OpenAIChat( # model_name="gpt-4" openai_api_key=api_key, temperature=0.5, - #max_tokens=100, + # max_tokens=100, ) ## Initialize the workflow diff --git a/groupchat.py b/groupchat.py index a97fbdd4..739181d1 100644 --- a/groupchat.py +++ b/groupchat.py @@ -15,28 +15,28 @@ flow1 = Flow( llm=llm, max_loops=1, system_message="YOU ARE SILLY, YOU OFFER NOTHING OF VALUE", - name='silly', + name="silly", dashboard=True, ) flow2 = Flow( llm=llm, max_loops=1, system_message="YOU ARE VERY SMART AND ANSWER RIDDLES", - name='detective', + name="detective", dashboard=True, ) flow3 = Flow( llm=llm, max_loops=1, system_message="YOU MAKE RIDDLES", - name='riddler', + name="riddler", dashboard=True, ) manager = Flow( llm=llm, max_loops=1, system_message="YOU ARE A GROUP CHAT MANAGER", - name='manager', + name="manager", dashboard=True, ) @@ -45,5 +45,5 @@ manager = Flow( agents = [flow1, flow2, flow3] group_chat = GroupChat(agents=agents, messages=[], max_round=10) -chat_manager = GroupChatManager(groupchat=group_chat, selector = manager) -chat_history = chat_manager("Write me a riddle") \ No newline at end of file +chat_manager = GroupChatManager(groupchat=group_chat, selector=manager) +chat_history = chat_manager("Write me a riddle") diff --git a/swarms/models/__init__.py b/swarms/models/__init__.py index 328dd013..4cb61b9a 100644 --- a/swarms/models/__init__.py +++ b/swarms/models/__init__.py @@ -16,6 +16,7 @@ from swarms.models.kosmos_two import Kosmos from swarms.models.vilt import Vilt from swarms.models.nougat import Nougat from swarms.models.layoutlm_document_qa import LayoutLMDocumentQA +# from swarms.models.distilled_whisperx import DistilWhisperModel # from swarms.models.fuyu import Fuyu # Not working, wait until they update import sys diff --git a/swarms/models/distilled_whisperx.py b/swarms/models/distilled_whisperx.py index 8062daa4..0a60aaac 100644 --- a/swarms/models/distilled_whisperx.py +++ b/swarms/models/distilled_whisperx.py @@ -1,3 +1,160 @@ -""" +import asyncio +import os +import time +from functools import wraps +from typing import Union -""" +import torch +from termcolor import colored +from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline + + +def async_retry(max_retries=3, exceptions=(Exception,), delay=1): + """ + A decorator for adding retry logic to async functions. + :param max_retries: Maximum number of retries before giving up. + :param exceptions: A tuple of exceptions to catch and retry on. + :param delay: Delay between retries. + """ + + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + retries = max_retries + while retries: + try: + return await func(*args, **kwargs) + except exceptions as e: + retries -= 1 + if retries <= 0: + raise + print(f"Retry after exception: {e}, Attempts remaining: {retries}") + await asyncio.sleep(delay) + + return wrapper + + return decorator + + +class DistilWhisperModel: + """ + This class encapsulates the Distil-Whisper model for English speech recognition. + It allows for both synchronous and asynchronous transcription of short and long-form audio. + + Args: + model_id: The model ID to use. Defaults to "distil-whisper/distil-large-v2". + + + Attributes: + device: The device to use for inference. + torch_dtype: The torch data type to use for inference. + model_id: The model ID to use. + model: The model instance. + processor: The processor instance. + + Usage: + model_wrapper = DistilWhisperModel() + transcription = model_wrapper('path/to/audio.mp3') + + # For async usage + transcription = asyncio.run(model_wrapper.async_transcribe('path/to/audio.mp3')) + """ + + def __init__(self, model_id="distil-whisper/distil-large-v2"): + self.device = "cuda:0" if torch.cuda.is_available() else "cpu" + self.torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32 + self.model_id = model_id + self.model = AutoModelForSpeechSeq2Seq.from_pretrained( + model_id, + torch_dtype=self.torch_dtype, + low_cpu_mem_usage=True, + use_safetensors=True, + ).to(self.device) + self.processor = AutoProcessor.from_pretrained(model_id) + + def __call__(self, inputs: Union[str, dict]): + return self.transcribe(inputs) + + def transcribe(self, inputs: Union[str, dict]): + """ + Synchronously transcribe the given audio input using the Distil-Whisper model. + :param inputs: A string representing the file path or a dict with audio data. + :return: The transcribed text. + """ + pipe = pipeline( + "automatic-speech-recognition", + model=self.model, + tokenizer=self.processor.tokenizer, + feature_extractor=self.processor.feature_extractor, + max_new_tokens=128, + torch_dtype=self.torch_dtype, + device=self.device, + ) + + return pipe(inputs)["text"] + + @async_retry() + async def async_transcribe(self, inputs: Union[str, dict]): + """ + Asynchronously transcribe the given audio input using the Distil-Whisper model. + :param inputs: A string representing the file path or a dict with audio data. + :return: The transcribed text. + """ + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self.transcribe, inputs) + + def real_time_transcribe(self, audio_file_path, chunk_duration=5): + """ + Simulates real-time transcription of an audio file, processing and printing results + in chunks with colored output for readability. + + :param audio_file_path: Path to the audio file to be transcribed. + :param chunk_duration: Duration in seconds of each audio chunk to be processed. + """ + if not os.path.isfile(audio_file_path): + print(colored("The audio file was not found.", "red")) + return + + # Assuming `chunk_duration` is in seconds and `processor` can handle chunk-wise processing + try: + with torch.no_grad(): + # Load the whole audio file, but process and transcribe it in chunks + audio_input = self.processor.audio_file_to_array(audio_file_path) + sample_rate = audio_input.sampling_rate + total_duration = len(audio_input.array) / sample_rate + chunks = [ + audio_input.array[i : i + sample_rate * chunk_duration] + for i in range( + 0, len(audio_input.array), sample_rate * chunk_duration + ) + ] + + print(colored("Starting real-time transcription...", "green")) + + for i, chunk in enumerate(chunks): + # Process the current chunk + processed_inputs = self.processor( + chunk, + sampling_rate=sample_rate, + return_tensors="pt", + padding=True, + ) + processed_inputs = processed_inputs.input_values.to(self.device) + + # Generate transcription for the chunk + logits = self.model.generate(processed_inputs) + transcription = self.processor.batch_decode( + logits, skip_special_tokens=True + )[0] + + # Print the chunk's transcription + print( + colored(f"Chunk {i+1}/{len(chunks)}: ", "yellow") + + transcription + ) + + # Wait for the chunk's duration to simulate real-time processing + time.sleep(chunk_duration) + + except Exception as e: + print(colored(f"An error occurred during transcription: {e}", "red")) From 7e1d486a024024d3a05bfe6eefc09b36ce6a5600 Mon Sep 17 00:00:00 2001 From: Kye Date: Sat, 4 Nov 2023 17:05:06 -0400 Subject: [PATCH 08/32] tests for distilled whisperx --- swarms/models/__init__.py | 1 + tests/models/distilled_whisperx.py | 120 +++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 tests/models/distilled_whisperx.py diff --git a/swarms/models/__init__.py b/swarms/models/__init__.py index 4cb61b9a..a0bec07f 100644 --- a/swarms/models/__init__.py +++ b/swarms/models/__init__.py @@ -16,6 +16,7 @@ from swarms.models.kosmos_two import Kosmos from swarms.models.vilt import Vilt from swarms.models.nougat import Nougat from swarms.models.layoutlm_document_qa import LayoutLMDocumentQA + # from swarms.models.distilled_whisperx import DistilWhisperModel # from swarms.models.fuyu import Fuyu # Not working, wait until they update diff --git a/tests/models/distilled_whisperx.py b/tests/models/distilled_whisperx.py new file mode 100644 index 00000000..bab8cd0e --- /dev/null +++ b/tests/models/distilled_whisperx.py @@ -0,0 +1,120 @@ +# test_distilled_whisperx.py + +from unittest.mock import AsyncMock, MagicMock + +import pytest +import torch +from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor + +from swarms.models.distilled_whisperx import DistilWhisperModel, async_retry + + +# Fixtures for setting up model, processor, and audio files +@pytest.fixture(scope="module") +def model_id(): + return "distil-whisper/distil-large-v2" + + +@pytest.fixture(scope="module") +def whisper_model(model_id): + return DistilWhisperModel(model_id) + + +@pytest.fixture(scope="session") +def audio_file_path(tmp_path_factory): + # You would create a small temporary MP3 file here for testing + # or use a public domain MP3 file's path + return "path/to/valid_audio.mp3" + + +@pytest.fixture(scope="session") +def invalid_audio_file_path(): + return "path/to/invalid_audio.mp3" + + +@pytest.fixture(scope="session") +def audio_dict(): + # This should represent a valid audio dictionary as expected by the model + return {"array": torch.randn(1, 16000), "sampling_rate": 16000} + + +# Test initialization +def test_initialization(whisper_model): + assert whisper_model.model is not None + assert whisper_model.processor is not None + + +# Test successful transcription with file path +def test_transcribe_with_file_path(whisper_model, audio_file_path): + transcription = whisper_model.transcribe(audio_file_path) + assert isinstance(transcription, str) + + +# Test successful transcription with audio dict +def test_transcribe_with_audio_dict(whisper_model, audio_dict): + transcription = whisper_model.transcribe(audio_dict) + assert isinstance(transcription, str) + + +# Test for file not found error +def test_file_not_found(whisper_model, invalid_audio_file_path): + with pytest.raises(Exception): + whisper_model.transcribe(invalid_audio_file_path) + + +# Asynchronous tests +@pytest.mark.asyncio +async def test_async_transcription_success(whisper_model, audio_file_path): + transcription = await whisper_model.async_transcribe(audio_file_path) + assert isinstance(transcription, str) + + +@pytest.mark.asyncio +async def test_async_transcription_failure(whisper_model, invalid_audio_file_path): + with pytest.raises(Exception): + await whisper_model.async_transcribe(invalid_audio_file_path) + + +# Testing real-time transcription simulation +def test_real_time_transcription(whisper_model, audio_file_path, capsys): + whisper_model.real_time_transcribe(audio_file_path, chunk_duration=1) + captured = capsys.readouterr() + assert "Starting real-time transcription..." in captured.out + + +# Testing retry decorator for asynchronous function +@pytest.mark.asyncio +async def test_async_retry(): + @async_retry(max_retries=2, exceptions=(ValueError,), delay=0) + async def failing_func(): + raise ValueError("Test") + + with pytest.raises(ValueError): + await failing_func() + + +# Mocking the actual model to avoid GPU/CPU intensive operations during test +@pytest.fixture +def mocked_model(monkeypatch): + model_mock = AsyncMock(AutoModelForSpeechSeq2Seq) + processor_mock = MagicMock(AutoProcessor) + monkeypatch.setattr( + "swarms.models.distilled_whisperx.AutoModelForSpeechSeq2Seq.from_pretrained", + model_mock, + ) + monkeypatch.setattr( + "swarms.models.distilled_whisperx.AutoProcessor.from_pretrained", processor_mock + ) + return model_mock, processor_mock + + +@pytest.mark.asyncio +async def test_async_transcribe_with_mocked_model(mocked_model, audio_file_path): + model_mock, processor_mock = mocked_model + # Set up what the mock should return when it's called + model_mock.return_value.generate.return_value = torch.tensor([[0]]) + processor_mock.return_value.batch_decode.return_value = ["mocked transcription"] + model_wrapper = DistilWhisperModel() + transcription = await model_wrapper.async_transcribe(audio_file_path) + assert transcription == "mocked transcription" + From 6e6fe8dc52b5fc79c972017c2d48ea3eae1138ca Mon Sep 17 00:00:00 2001 From: Kye Date: Sat, 4 Nov 2023 21:09:04 -0400 Subject: [PATCH 09/32] docs for DistilWhisperModel --- docs/swarms/models/distilled_whisperx.md | 123 +++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 124 insertions(+) create mode 100644 docs/swarms/models/distilled_whisperx.md diff --git a/docs/swarms/models/distilled_whisperx.md b/docs/swarms/models/distilled_whisperx.md new file mode 100644 index 00000000..e9339c1e --- /dev/null +++ b/docs/swarms/models/distilled_whisperx.md @@ -0,0 +1,123 @@ +# DistilWhisperModel Documentation + +## Overview + +The `DistilWhisperModel` is a Python class designed to handle English speech recognition tasks. It leverages the capabilities of the Whisper model, which is fine-tuned for speech-to-text processes. It is designed for both synchronous and asynchronous transcription of audio inputs, offering flexibility for real-time applications or batch processing. + +## Installation + +Before you can use `DistilWhisperModel`, ensure you have the required libraries installed: + +```sh +pip3 install --upgrade swarms +``` + +## Initialization + +The `DistilWhisperModel` class is initialized with the following parameters: + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `model_id` | `str` | The identifier for the pre-trained Whisper model | `"distil-whisper/distil-large-v2"` | + +Example of initialization: + +```python +from swarms.models import DistilWhisperModel + +# Initialize with default model +model_wrapper = DistilWhisperModel() + +# Initialize with a specific model ID +model_wrapper = DistilWhisperModel(model_id='distil-whisper/distil-large-v2') +``` + +## Attributes + +After initialization, the `DistilWhisperModel` has several attributes: + +| Attribute | Type | Description | +|-----------|------|-------------| +| `device` | `str` | The device used for computation (`"cuda:0"` for GPU or `"cpu"`). | +| `torch_dtype` | `torch.dtype` | The data type used for the Torch tensors. | +| `model_id` | `str` | The model identifier string. | +| `model` | `torch.nn.Module` | The actual Whisper model loaded from the identifier. | +| `processor` | `transformers.AutoProcessor` | The processor for handling input data. | + +## Methods + +### `transcribe` + +Transcribes audio input synchronously. + +**Arguments**: + +| Argument | Type | Description | +|----------|------|-------------| +| `inputs` | `Union[str, dict]` | File path or audio data dictionary. | + +**Returns**: `str` - The transcribed text. + +**Usage Example**: + +```python +# Synchronous transcription +transcription = model_wrapper.transcribe('path/to/audio.mp3') +print(transcription) +``` + +### `async_transcribe` + +Transcribes audio input asynchronously. + +**Arguments**: + +| Argument | Type | Description | +|----------|------|-------------| +| `inputs` | `Union[str, dict]` | File path or audio data dictionary. | + +**Returns**: `Coroutine` - A coroutine that when awaited, returns the transcribed text. + +**Usage Example**: + +```python +import asyncio + +# Asynchronous transcription +transcription = asyncio.run(model_wrapper.async_transcribe('path/to/audio.mp3')) +print(transcription) +``` + +### `real_time_transcribe` + +Simulates real-time transcription of an audio file. + +**Arguments**: + +| Argument | Type | Description | +|----------|------|-------------| +| `audio_file_path` | `str` | Path to the audio file. | +| `chunk_duration` | `int` | Duration of audio chunks in seconds. | + +**Usage Example**: + +```python +# Real-time transcription simulation +model_wrapper.real_time_transcribe('path/to/audio.mp3', chunk_duration=5) +``` + +## Error Handling + +The `DistilWhisperModel` class incorporates error handling for file not found errors and generic exceptions during the transcription process. If a non-recoverable exception is raised, it is printed to the console in red to indicate failure. + +## Conclusion + +The `DistilWhisperModel` offers a convenient interface to the powerful Whisper model for speech recognition. Its design supports both batch and real-time transcription, catering to different application needs. The class's error handling and retry logic make it robust for real-world applications. + +## Additional Notes + +- Ensure you have appropriate permissions to read audio files when using file paths. +- Transcription quality depends on the audio quality and the Whisper model's performance on your dataset. +- Adjust `chunk_duration` according to the processing power of your system for real-time transcription. + +For a full list of models supported by `transformers.AutoModelForSpeechSeq2Seq`, visit the [Hugging Face Model Hub](https://huggingface.co/models). diff --git a/mkdocs.yml b/mkdocs.yml index bf155336..55c7cf3d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -106,6 +106,7 @@ nav: - Kosmos: "swarms/models/kosmos.md" - Nougat: "swarms/models/nougat.md" - LayoutLMDocumentQA: "swarms/models/layoutlm_document_qa.md" + - DistilWhisperModel: "swarms/models/distilled_whisperx.md" - swarms.structs: - Overview: "swarms/structs/overview.md" - Workflow: "swarms/structs/workflow.md" From 1fb193288b7017b0ae6bf5f91adc9b492820e9e0 Mon Sep 17 00:00:00 2001 From: Kye Date: Sat, 4 Nov 2023 21:12:31 -0400 Subject: [PATCH 10/32] fuyu fix --- swarms/models/fuyu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarms/models/fuyu.py b/swarms/models/fuyu.py index e8d16cdf..bdd3f904 100644 --- a/swarms/models/fuyu.py +++ b/swarms/models/fuyu.py @@ -61,6 +61,6 @@ class Fuyu: model_inputs[k] = v.to(self.device_map) output = self.model.generate( - **model_inputs, max_new_tokens=self.fmax_new_tokens + **model_inputs, max_new_tokens=self.max_new_tokens ) text = self.processor.batch_decode(output[:, -7:], skip_special_tokens=True) From d4bd4fa4a47eaeba44a08164c0f464f3aaa24dcb Mon Sep 17 00:00:00 2001 From: Kye Date: Sat, 4 Nov 2023 21:26:07 -0400 Subject: [PATCH 11/32] anthropic tests --- swarms/models/anthropic.py | 24 +++++++- tests/models/anthropic.py | 116 +++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 tests/models/anthropic.py diff --git a/swarms/models/anthropic.py b/swarms/models/anthropic.py index 232ff647..e2066637 100644 --- a/swarms/models/anthropic.py +++ b/swarms/models/anthropic.py @@ -7,9 +7,31 @@ class Anthropic: Anthropic large language models. - Args: + model: The model to use. Defaults to "claude-2". + max_tokens_to_sample: The maximum number of tokens to sample. + temperature: The temperature to use for sampling. + top_k: The top_k to use for sampling. + top_p: The top_p to use for sampling. + streaming: Whether to stream the response or not. + default_request_timeout: The default request timeout to use. + + + Attributes: + model: The model to use. + max_tokens_to_sample: The maximum number of tokens to sample. + temperature: The temperature to use for sampling. + top_k: The top_k to use for sampling. + top_p: The top_p to use for sampling. + streaming: Whether to stream the response or not. + default_request_timeout: The default request timeout to use. + anthropic_api_url: The API URL to use. + anthropic_api_key: The API key to use. + Usage: + model_wrapper = Anthropic() + completion = model_wrapper("Hello, my name is") + print(completion) """ diff --git a/tests/models/anthropic.py b/tests/models/anthropic.py new file mode 100644 index 00000000..844415aa --- /dev/null +++ b/tests/models/anthropic.py @@ -0,0 +1,116 @@ +import os +import pytest +from unittest.mock import Mock, patch +from swarms.models.anthropic import Anthropic + +@pytest.fixture +def mock_anthropic_env(): + os.environ["ANTHROPIC_API_URL"] = "https://test.anthropic.com" + os.environ["ANTHROPIC_API_KEY"] = "test_api_key" + yield + del os.environ["ANTHROPIC_API_URL"] + del os.environ["ANTHROPIC_API_KEY"] + +@pytest.fixture +def mock_requests_post(): + with patch("requests.post") as mock_post: + yield mock_post + +@pytest.fixture +def anthropic_instance(): + return Anthropic(model="test-model") + +def test_anthropic_init_default_values(anthropic_instance): + assert anthropic_instance.model == "test-model" + assert anthropic_instance.max_tokens_to_sample == 256 + assert anthropic_instance.temperature is None + assert anthropic_instance.top_k is None + assert anthropic_instance.top_p is None + assert anthropic_instance.streaming is False + assert anthropic_instance.default_request_timeout == 600 + assert anthropic_instance.anthropic_api_url == "https://test.anthropic.com" + assert anthropic_instance.anthropic_api_key == "test_api_key" + +def test_anthropic_init_custom_values(): + anthropic_instance = Anthropic( + model="custom-model", + max_tokens_to_sample=128, + temperature=0.8, + top_k=5, + top_p=0.9, + streaming=True, + default_request_timeout=300, + ) + assert anthropic_instance.model == "custom-model" + assert anthropic_instance.max_tokens_to_sample == 128 + assert anthropic_instance.temperature == 0.8 + assert anthropic_instance.top_k == 5 + assert anthropic_instance.top_p == 0.9 + assert anthropic_instance.streaming is True + assert anthropic_instance.default_request_timeout == 300 + +def test_anthropic_default_params(anthropic_instance): + default_params = anthropic_instance._default_params() + assert default_params == { + "max_tokens_to_sample": 256, + "model": "test-model", + } + +def test_anthropic_run(mock_anthropic_env, mock_requests_post, anthropic_instance): + mock_response = Mock() + mock_response.json.return_value = {"completion": "Generated text"} + mock_requests_post.return_value = mock_response + + task = "Generate text" + stop = ["stop1", "stop2"] + + completion = anthropic_instance.run(task, stop) + + assert completion == "Generated text" + mock_requests_post.assert_called_once_with( + "https://test.anthropic.com/completions", + headers={"Authorization": "Bearer test_api_key"}, + json={ + "prompt": task, + "stop_sequences": stop, + "max_tokens_to_sample": 256, + "model": "test-model", + }, + timeout=600, + ) + +def test_anthropic_call(mock_anthropic_env, mock_requests_post, anthropic_instance): + mock_response = Mock() + mock_response.json.return_value = {"completion": "Generated text"} + mock_requests_post.return_value = mock_response + + task = "Generate text" + stop = ["stop1", "stop2"] + + completion = anthropic_instance(task, stop) + + assert completion == "Generated text" + mock_requests_post.assert_called_once_with( + "https://test.anthropic.com/completions", + headers={"Authorization": "Bearer test_api_key"}, + json={ + "prompt": task, + "stop_sequences": stop, + "max_tokens_to_sample": 256, + "model": "test-model", + }, + timeout=600, + ) + +def test_anthropic_exception_handling(mock_anthropic_env, mock_requests_post, anthropic_instance): + mock_response = Mock() + mock_response.json.return_value = {"error": "An error occurred"} + mock_requests_post.return_value = mock_response + + task = "Generate text" + stop = ["stop1", "stop2"] + + with pytest.raises(Exception) as excinfo: + anthropic_instance(task, stop) + + assert "An error occurred" in str(excinfo.value) From ba28f40e579861e8e1bb524f15c2866599b66d7d Mon Sep 17 00:00:00 2001 From: Kye Date: Sun, 5 Nov 2023 10:37:10 -0500 Subject: [PATCH 12/32] auto saved + fixed run method of flow --- demos/positive_med.py | 4 +-- example.py | 8 ++--- flow_state.json | 14 -------- swarms/models/fuyu.py | 4 +-- swarms/structs/flow.py | 52 ++++++++++++++++-------------- tests/models/anthropic.py | 13 +++++++- tests/models/distilled_whisperx.py | 1 - 7 files changed, 47 insertions(+), 49 deletions(-) delete mode 100644 flow_state.json diff --git a/demos/positive_med.py b/demos/positive_med.py index e8f879c9..2d191c55 100644 --- a/demos/positive_med.py +++ b/demos/positive_med.py @@ -23,7 +23,7 @@ Distribution Agent: """ -from swarms import OpenAIChat +from swarms.models import OpenAIChat from termcolor import colored TOPIC_GENERATOR = f""" @@ -264,7 +264,7 @@ Denote the social media's by using the social media name in HTML like tags {{ARTICLE}} """ -llm = OpenAIChat(openai_api_key="") +llm = OpenAIChat(openai_api_key="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") def get_review_prompt(article): diff --git a/example.py b/example.py index aeae1b02..3af9fc57 100644 --- a/example.py +++ b/example.py @@ -1,7 +1,7 @@ from swarms.models import OpenAIChat from swarms.structs import Flow -api_key = "" +api_key = "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" # Initialize the language model, this model can be swapped out with Anthropic, ETC, Huggingface Models like Mistral, ETC llm = OpenAIChat( @@ -14,7 +14,7 @@ llm = OpenAIChat( ## Initialize the workflow flow = Flow( llm=llm, - max_loops=1, + max_loops=2, dashboard=True, # stopping_condition=None, # You can define a stopping condition as needed. # loop_interval=1, @@ -31,5 +31,5 @@ out = flow.run("Generate a 10,000 word blog on health and wellness.") # out = flow.validate_response(out) # out = flow.analyze_feedback(out) # out = flow.print_history_and_memory() -# out = flow.save_state("flow_state.json") -print(out) +# # out = flow.save_state("flow_state.json") +# print(out) diff --git a/flow_state.json b/flow_state.json deleted file mode 100644 index 8ed134a0..00000000 --- a/flow_state.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "memory": [ - [ - "Human: Generate a 10,000 word blog on health and wellness." - ] - ], - "llm_params": {}, - "loop_interval": 1, - "retry_attempts": 3, - "retry_interval": 1, - "interactive": false, - "dashboard": true, - "dynamic_temperature": false -} \ No newline at end of file diff --git a/swarms/models/fuyu.py b/swarms/models/fuyu.py index bdd3f904..0fd1fd85 100644 --- a/swarms/models/fuyu.py +++ b/swarms/models/fuyu.py @@ -60,7 +60,5 @@ class Fuyu: for k, v in model_inputs.items(): model_inputs[k] = v.to(self.device_map) - output = self.model.generate( - **model_inputs, max_new_tokens=self.max_new_tokens - ) + output = self.model.generate(**model_inputs, max_new_tokens=self.max_new_tokens) text = self.processor.batch_decode(output[:, -7:], skip_special_tokens=True) diff --git a/swarms/structs/flow.py b/swarms/structs/flow.py index afbcf536..d40e4fb4 100644 --- a/swarms/structs/flow.py +++ b/swarms/structs/flow.py @@ -105,6 +105,8 @@ class Flow: system_message: str = FLOW_SYSTEM_PROMPT, # tools: List[BaseTool] = None, dynamic_temperature: bool = False, + saved_state: Optional[str] = None, + autosave: bool = False, **kwargs: Any, ): self.llm = llm @@ -124,6 +126,9 @@ class Flow: # self.tools = tools self.system_message = system_message self.name = name + self.saved_state = saved_state + self.autosave = autosave + self.response_filters = [] def provide_feedback(self, feedback: str) -> None: """Allow users to provide feedback on the responses.""" @@ -206,7 +211,7 @@ class Flow: print(dashboard) - def run(self, task: str, save: bool = True, **kwargs): + def run(self, task: str, **kwargs): """ Run the autonomous agent loop @@ -220,15 +225,15 @@ class Flow: 4. If stopping condition is not met, generate a response 5. Repeat until stopping condition is met or max_loops is reached - Example: - >>> out = flow.run("Generate a 10,000 word blog on health and wellness.") - """ - # Start with a new history or continue from the last saved state - if not self.memory or not self.memory[-1]: - history = [f"Human: {task}"] - else: - history = self.memory[-1] + # Restore from saved state if provided, ortherwise start with a new history + # if self.saved_state: + # self.load_state(self.saved_state) + # history = self.memory[-1] + # print(f"Loaded state from {self.saved_state}") + # else: + # history = [f"Human: {task}"] + # self.memory.append(history) response = task history = [f"Human: {task}"] @@ -237,12 +242,9 @@ class Flow: if self.dashboard: self.print_dashboard(task) - # Start or continue the loop process - for i in range(len(history), self.max_loops): + for i in range(self.max_loops): print(colored(f"\nLoop {i+1} of {self.max_loops}", "blue")) print("\n") - response = history[-1].split(": ", 1)[-1] # Get the last response - if self._check_stopping_condition(response) or parse_done_token(response): break @@ -254,8 +256,8 @@ class Flow: while attempt < self.retry_attempts: try: response = self.llm( - self.agent_history_prompt(FLOW_SYSTEM_PROMPT, response) - ** kwargs, + self.agent_history_prompt(FLOW_SYSTEM_PROMPT, response), + **kwargs, ) # print(f"Next query: {response}") # break @@ -277,8 +279,8 @@ class Flow: time.sleep(self.loop_interval) self.memory.append(history) - if save: - self.save("flow_history.json") + # if self.autosave: + # self.save_state("flow_state.json") return response # , history @@ -353,8 +355,8 @@ class Flow: time.sleep(self.loop_interval) self.memory.append(history) - if save: - self.save_state("flow_history.json") + # if save: + # self.save_state("flow_history.json") return response # , history @@ -409,7 +411,13 @@ class Flow: json.dump(self.memory, f) print(f"Saved flow history to {file_path}") - def load(self, file_path) -> None: + def load(self, file_path: str): + """ + Load the flow history from a file. + + Args: + file_path (str): The path to the file containing the saved flow history. + """ with open(file_path, "r") as f: self.memory = json.load(f) print(f"Loaded flow history from {file_path}") @@ -660,10 +668,6 @@ class Flow: with open(file_path, "r") as f: state = json.load(f) - # Assuming 'llm_class' is a class reference to the language - # llm_params = state.get("llm_params", {}) - # self.llm = self.llm(**llm_params) - # Restore other saved attributes self.memory = state.get("memory", []) self.max_loops = state.get("max_loops", 5) diff --git a/tests/models/anthropic.py b/tests/models/anthropic.py index 844415aa..4dbd365d 100644 --- a/tests/models/anthropic.py +++ b/tests/models/anthropic.py @@ -3,6 +3,7 @@ import pytest from unittest.mock import Mock, patch from swarms.models.anthropic import Anthropic + @pytest.fixture def mock_anthropic_env(): os.environ["ANTHROPIC_API_URL"] = "https://test.anthropic.com" @@ -11,15 +12,18 @@ def mock_anthropic_env(): del os.environ["ANTHROPIC_API_URL"] del os.environ["ANTHROPIC_API_KEY"] + @pytest.fixture def mock_requests_post(): with patch("requests.post") as mock_post: yield mock_post + @pytest.fixture def anthropic_instance(): return Anthropic(model="test-model") + def test_anthropic_init_default_values(anthropic_instance): assert anthropic_instance.model == "test-model" assert anthropic_instance.max_tokens_to_sample == 256 @@ -31,6 +35,7 @@ def test_anthropic_init_default_values(anthropic_instance): assert anthropic_instance.anthropic_api_url == "https://test.anthropic.com" assert anthropic_instance.anthropic_api_key == "test_api_key" + def test_anthropic_init_custom_values(): anthropic_instance = Anthropic( model="custom-model", @@ -49,6 +54,7 @@ def test_anthropic_init_custom_values(): assert anthropic_instance.streaming is True assert anthropic_instance.default_request_timeout == 300 + def test_anthropic_default_params(anthropic_instance): default_params = anthropic_instance._default_params() assert default_params == { @@ -56,6 +62,7 @@ def test_anthropic_default_params(anthropic_instance): "model": "test-model", } + def test_anthropic_run(mock_anthropic_env, mock_requests_post, anthropic_instance): mock_response = Mock() mock_response.json.return_value = {"completion": "Generated text"} @@ -79,6 +86,7 @@ def test_anthropic_run(mock_anthropic_env, mock_requests_post, anthropic_instanc timeout=600, ) + def test_anthropic_call(mock_anthropic_env, mock_requests_post, anthropic_instance): mock_response = Mock() mock_response.json.return_value = {"completion": "Generated text"} @@ -102,7 +110,10 @@ def test_anthropic_call(mock_anthropic_env, mock_requests_post, anthropic_instan timeout=600, ) -def test_anthropic_exception_handling(mock_anthropic_env, mock_requests_post, anthropic_instance): + +def test_anthropic_exception_handling( + mock_anthropic_env, mock_requests_post, anthropic_instance +): mock_response = Mock() mock_response.json.return_value = {"error": "An error occurred"} mock_requests_post.return_value = mock_response diff --git a/tests/models/distilled_whisperx.py b/tests/models/distilled_whisperx.py index bab8cd0e..4bdd10f3 100644 --- a/tests/models/distilled_whisperx.py +++ b/tests/models/distilled_whisperx.py @@ -117,4 +117,3 @@ async def test_async_transcribe_with_mocked_model(mocked_model, audio_file_path) model_wrapper = DistilWhisperModel() transcription = await model_wrapper.async_transcribe(audio_file_path) assert transcription == "mocked transcription" - From 310230a417c7ae78dea19b5d11d3a20033c9993f Mon Sep 17 00:00:00 2001 From: Kye Date: Sun, 5 Nov 2023 21:46:28 -0500 Subject: [PATCH 13/32] sequential workflow tests, prorotytpe with documentation --- demos/positive_med.py | 2 +- docs/swarms/structs/sequential_workflow.md | 577 +++++++++++++++++++++ example.py | 2 +- sequential_workflow_example.py | 37 ++ swarms/structs/flow.py | 94 +++- swarms/structs/sequential_workflow.py | 419 ++++++++++++--- swarms/swarms/autobloggen.py | 0 tests/structs/sequential_workflow.py | 306 +++++++++++ 8 files changed, 1348 insertions(+), 89 deletions(-) create mode 100644 docs/swarms/structs/sequential_workflow.md create mode 100644 sequential_workflow_example.py create mode 100644 swarms/swarms/autobloggen.py create mode 100644 tests/structs/sequential_workflow.py diff --git a/demos/positive_med.py b/demos/positive_med.py index 2d191c55..88226545 100644 --- a/demos/positive_med.py +++ b/demos/positive_med.py @@ -264,7 +264,7 @@ Denote the social media's by using the social media name in HTML like tags {{ARTICLE}} """ -llm = OpenAIChat(openai_api_key="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") +llm = OpenAIChat(openai_api_key="") def get_review_prompt(article): diff --git a/docs/swarms/structs/sequential_workflow.md b/docs/swarms/structs/sequential_workflow.md new file mode 100644 index 00000000..04587b89 --- /dev/null +++ b/docs/swarms/structs/sequential_workflow.md @@ -0,0 +1,577 @@ +# `SequentialWorkflow` Documentation + +The **SequentialWorkflow** class is a Python module designed to facilitate the execution of a sequence of tasks in a sequential manner. It is a part of the `swarms.structs` package and is particularly useful for orchestrating the execution of various callable objects, such as functions or models, in a predefined order. This documentation will provide an in-depth understanding of the **SequentialWorkflow** class, including its purpose, architecture, usage, and examples. + +## Purpose and Relevance + +The **SequentialWorkflow** class is essential for managing and executing a series of tasks or processes, where each task may depend on the outcome of the previous one. It is commonly used in various application scenarios, including but not limited to: + +1. **Natural Language Processing (NLP) Workflows:** In NLP workflows, multiple language models are employed sequentially to process and generate text. Each model may depend on the results of the previous one, making sequential execution crucial. + +2. **Data Analysis Pipelines:** Data analysis often involves a series of tasks such as data preprocessing, transformation, and modeling steps. These tasks must be performed sequentially to ensure data consistency and accuracy. + +3. **Task Automation:** In task automation scenarios, there is a need to execute a series of automated tasks in a specific order. Sequential execution ensures that each task is performed in a predefined sequence, maintaining the workflow's integrity. + +By providing a structured approach to managing these tasks, the **SequentialWorkflow** class helps developers streamline their workflow execution and improve code maintainability. + +## Key Concepts and Terminology + +Before delving into the details of the **SequentialWorkflow** class, let's define some key concepts and terminology that will be used throughout the documentation: + +### Task + +A **task** refers to a specific unit of work that needs to be executed as part of the workflow. Each task is associated with a description and can be implemented as a callable object, such as a function or a model. + +### Flow + +A **flow** represents a callable object that can be a task within the **SequentialWorkflow**. Flows encapsulate the logic and functionality of a particular task. Flows can be functions, models, or any callable object that can be executed. + +### Sequential Execution + +Sequential execution refers to the process of running tasks one after the other in a predefined order. In a **SequentialWorkflow**, tasks are executed sequentially, meaning that each task starts only after the previous one has completed. + +### Workflow + +A **workflow** is a predefined sequence of tasks that need to be executed in a specific order. It represents the overall process or pipeline that the **SequentialWorkflow** manages. + +### Dashboard (Optional) + +A **dashboard** is an optional feature of the **SequentialWorkflow** that provides real-time monitoring and visualization of the workflow's progress. It displays information such as the current task being executed, task results, and other relevant metadata. + +### Max Loops + +The **maximum number of times** the entire workflow can be run. This parameter allows developers to control how many times the workflow is executed. + +### Autosaving + +**Autosaving** is a feature that allows the **SequentialWorkflow** to automatically save its state to a file at specified intervals. This feature helps in resuming a workflow from where it left off, even after interruptions. + +Now that we have a clear understanding of the key concepts and terminology, let's explore the architecture and usage of the **SequentialWorkflow** class in more detail. + +## Architecture of SequentialWorkflow + +The architecture of the **SequentialWorkflow** class is designed to provide a structured and flexible way to define, manage, and execute a sequence of tasks. It comprises the following core components: + +1. **Task**: The **Task** class represents an individual unit of work within the workflow. Each task has a description, which serves as a human-readable identifier for the task. Tasks can be implemented as callable objects, allowing for great flexibility in defining their functionality. + +2. **Workflow**: The **SequentialWorkflow** class itself represents the workflow. It manages a list of tasks in the order they should be executed. Workflows can be run sequentially or asynchronously, depending on the use case. + +3. **Task Execution**: Task execution is the process of running each task in the workflow. Tasks are executed one after another in the order they were added to the workflow. Task results can be passed as inputs to subsequent tasks. + +4. **Dashboard (Optional)**: The **SequentialWorkflow** optionally includes a dashboard feature. The dashboard provides a visual interface for monitoring the progress of the workflow. It displays information about the current task, task results, and other relevant metadata. + +5. **State Management**: The **SequentialWorkflow** supports state management, allowing developers to save and load the state of the workflow to and from JSON files. This feature is valuable for resuming workflows after interruptions or for sharing workflow configurations. + +## Usage of SequentialWorkflow + +The **SequentialWorkflow** class is versatile and can be employed in a wide range of applications. Its usage typically involves the following steps: + +1. **Initialization**: Begin by initializing any callable objects or flows that will serve as tasks in the workflow. These callable objects can include functions, models, or any other Python objects that can be executed. + +2. **Workflow Creation**: Create an instance of the **SequentialWorkflow** class. Specify the maximum number of loops the workflow should run and whether a dashboard should be displayed. + +3. **Task Addition**: Add tasks to the workflow using the `add` method. Each task should be described using a human-readable description, and the associated flow (callable object) should be provided. Additional arguments and keyword arguments can be passed to the task. + +4. **Task Execution**: Execute the workflow using the `run` method. The tasks within the workflow will be executed sequentially, with task results passed as inputs to subsequent tasks. + +5. **Accessing Results**: After running the workflow, you can access the results of each task using the `get_task_results` method or by directly accessing the `result` attribute of each task. + +6. **Optional Features**: Optionally, you can enable features such as autosaving of the workflow state and utilize the dashboard for real-time monitoring. + + +## Installation + +Before using the Sequential Workflow library, you need to install it. You can install it via pip: + +```bash +pip3 install --upgrade swarms +``` + +## Quick Start + +Let's begin with a quick example to demonstrate how to create and run a Sequential Workflow. In this example, we'll create a workflow that generates a 10,000-word blog on "health and wellness" using an AI model and then summarizes the generated content. + +```python +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Initialize the language model flow (e.g., GPT-3) +llm = OpenAIChat( + openai_api_key="YOUR_API_KEY", + temperature=0.5, + max_tokens=3000, +) + +# Initialize flows for individual tasks +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the Sequential Workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) +workflow.add("Summarize the generated blog", flow2) + +# Run the workflow +workflow.run() + +# Output the results +for task in workflow.tasks: + print(f"Task: {task.description}, Result: {task.result}") +``` + +This quick example demonstrates the basic usage of the Sequential Workflow. It creates two tasks and executes them sequentially. + +## Class: `Task` + +### Description + +The `Task` class represents an individual task in the workflow. A task is essentially a callable object, such as a function or a class, that can be executed sequentially. Tasks can have arguments and keyword arguments. + +### Class Definition + +```python +class Task: + def __init__(self, description: str, flow: Union[Callable, Flow], args: List[Any] = [], kwargs: Dict[str, Any] = {}, result: Any = None, history: List[Any] = []) +``` + +### Parameters + +- `description` (str): A description of the task. +- `flow` (Union[Callable, Flow]): The callable object representing the task. It can be a function, class, or a `Flow` instance. +- `args` (List[Any]): A list of positional arguments to pass to the task when executed. Default is an empty list. +- `kwargs` (Dict[str, Any]): A dictionary of keyword arguments to pass to the task when executed. Default is an empty dictionary. +- `result` (Any): The result of the task's execution. Default is `None`. +- `history` (List[Any]): A list to store the historical results of the task. Default is an empty list. + +### Methods + +#### `execute()` + +Execute the task. + +```python +def execute(self): +``` + +This method executes the task and updates the `result` and `history` attributes of the task. It checks if the task is a `Flow` instance and if the 'task' argument is needed. + +## Class: `SequentialWorkflow` + +### Description + +The `SequentialWorkflow` class is responsible for managing a sequence of tasks and executing them in a sequential order. It provides methods for adding tasks, running the workflow, and managing the state of the tasks. + +### Class Definition + +```python +class SequentialWorkflow: + def __init__(self, max_loops: int = 1, autosave: bool = False, saved_state_filepath: Optional[str] = "sequential_workflow_state.json", restore_state_filepath: Optional[str] = None, dashboard: bool = False, tasks: List[Task] = []) +``` + +### Parameters + +- `max_loops` (int): The maximum number of times to run the workflow sequentially. Default is `1`. +- `autosave` (bool): Whether to enable autosaving of the workflow state. Default is `False`. +- `saved_state_filepath` (Optional[str]): The file path to save the workflow state when autosave is enabled. Default is `"sequential_workflow_state.json"`. +- `restore_state_filepath` (Optional[str]): The file path to restore the workflow state when initializing. Default is `None`. +- `dashboard` (bool): Whether to display a dashboard with workflow information. Default is `False`. +- `tasks` (List[Task]): A list of `Task` instances representing the tasks in the workflow. Default is an empty list. + +### Methods + +#### `add(task: str, flow: Union[Callable, Flow], *args, **kwargs)` + +Add a task to the workflow. + +```python +def add(self, task: str, flow: Union[Callable, Flow], *args, **kwargs) -> None: +``` + +This method adds a new task to the workflow. You can provide a description of the task, the callable object (function, class, or `Flow` instance), and any additional positional or keyword arguments required for the task. + +#### `reset_workflow()` + +Reset the workflow by clearing the results of each task. + +```python +def reset_workflow(self) -> None: +``` + +This method clears the results of each task in the workflow, allowing you to start fresh without reinitializing the workflow. + +#### `get_task_results()` + +Get the results of each task in the workflow. + +```python +def get_task_results(self) -> Dict[str, Any]: +``` + +This method returns a dictionary containing the results of each task in the workflow, where the keys are task descriptions, and the values are the corresponding results. + +#### `remove_task(task_description: str)` + +Remove a task from the workflow. + +```python +def remove_task(self, task_description: str) -> None: +``` + +This method removes a specific task from the workflow based on its description. + +#### `update_task(task_description: str, **updates)` + +Update the arguments of a task in the workflow. + +```python +def update_task(self, task_description: str, **updates) -> None: +``` + +This method allows you to update the arguments and keyword arguments of a task in the workflow. You specify the task's description and provide the updates as keyword arguments. + +#### `save_workflow_state(filepath: Optional[str] = "sequential_workflow_state.json", **kwargs)` + +Save the workflow state to a JSON file. + +```python +def save_workflow_state(self, filepath: Optional[str] = "sequential_workflow_state.json", **kwargs) -> None: +``` + +This method saves the current state of the workflow, including the results and history of each task, to a JSON file. You can specify the file path for saving the state. + +#### `load_workflow_state(filepath: str = None, **kwargs)` + +Load the workflow state from a JSON file and restore the workflow state. + +```python +def load_workflow_state(self, filepath: str = None, **kwargs) -> None: +``` + +This method loads a previously saved workflow state from a JSON file + + and restores the state, allowing you to continue the workflow from where it was saved. You can specify the file path for loading the state. + +#### `run()` + +Run the workflow sequentially. + +```python +def run(self) -> None: +``` + +This method executes the tasks in the workflow sequentially. It checks if a task is a `Flow` instance and handles the flow of data between tasks accordingly. + +#### `arun()` + +Asynchronously run the workflow. + +```python +async def arun(self) -> None: +``` + +This method asynchronously executes the tasks in the workflow sequentially. It's suitable for use cases where asynchronous execution is required. It also handles data flow between tasks. + +#### `workflow_bootup(**kwargs)` + +Display a bootup message for the workflow. + +```python +def workflow_bootup(self, **kwargs) -> None: +``` + +This method displays a bootup message when the workflow is initialized. You can customize the message by providing additional keyword arguments. + +#### `workflow_dashboard(**kwargs)` + +Display a dashboard for the workflow. + +```python +def workflow_dashboard(self, **kwargs) -> None: +``` + +This method displays a dashboard with information about the workflow, such as the number of tasks, maximum loops, and autosave settings. You can customize the dashboard by providing additional keyword arguments. + +## Examples + +Let's explore some examples to illustrate how to use the Sequential Workflow library effectively. + +Sure, I'll recreate the usage examples section for each method and use case using the provided foundation. Here are the examples: + +### Example 1: Adding Tasks to a Sequential Workflow + +In this example, we'll create a Sequential Workflow and add tasks to it. + +```python +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Example usage +api_key = ( + "" # Your actual API key here +) + +# Initialize the language flow +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) + +# Initialize Flows for individual tasks +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the Sequential Workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) +workflow.add("Summarize the generated blog", flow2) + +# Output the list of tasks in the workflow +print("Tasks in the workflow:") +for task in workflow.tasks: + print(f"Task: {task.description}") +``` + +In this example, we create a Sequential Workflow and add two tasks to it. + +### Example 2: Resetting a Sequential Workflow + +In this example, we'll create a Sequential Workflow, add tasks to it, and then reset it. + +```python +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Example usage +api_key = ( + "" # Your actual API key here +) + +# Initialize the language flow +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) + +# Initialize Flows for individual tasks +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the Sequential Workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) +workflow.add("Summarize the generated blog", flow2) + +# Reset the workflow +workflow.reset_workflow() + +# Output the list of tasks in the workflow after resetting +print("Tasks in the workflow after resetting:") +for task in workflow.tasks: + print(f"Task: {task.description}") +``` + +In this example, we create a Sequential Workflow, add two tasks to it, and then reset the workflow, clearing all task results. + +### Example 3: Getting Task Results from a Sequential Workflow + +In this example, we'll create a Sequential Workflow, add tasks to it, run the workflow, and then retrieve the results of each task. + +```python +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Example usage +api_key = ( + "" # Your actual API key here +) + +# Initialize the language flow +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) + +# Initialize Flows for individual tasks +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the Sequential Workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) +workflow.add("Summarize the generated blog", flow2) + +# Run the workflow +workflow.run() + +# Get and display the results of each task in the workflow +results = workflow.get_task_results() +for task_description, result in results.items(): + print(f"Task: {task_description}, Result: {result}") +``` + +In this example, we create a Sequential Workflow, add two tasks to it, run the workflow, and then retrieve and display the results of each task. + +### Example 4: Removing a Task from a Sequential Workflow + +In this example, we'll create a Sequential Workflow, add tasks to it, and then remove a specific task from the workflow. + +```python +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Example usage +api_key = ( + "" # Your actual API key here +) + +# Initialize the language flow +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) + +# Initialize Flows for individual tasks +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the Sequential Workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) +workflow.add("Summarize the generated blog", flow2) + +# Remove a specific task from the workflow +workflow.remove_task("Generate a 10,000 word blog on health and wellness.") + +# Output the list of tasks in the workflow after removal +print("Tasks in the workflow after removing a task:") +for task in workflow.tasks: + print(f"Task: {task.description}") +``` + +In this example, we create a Sequential Workflow, add two tasks to it, and then remove a specific task from the workflow. + +### Example 5: Updating Task Arguments in a Sequential Workflow + +In this example, we'll create a Sequential Workflow, add tasks to it, and then update the arguments of a specific task in the workflow. + +```python +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Example usage +api_key = ( + "" # Your actual API key here +) + +# Initialize the language flow +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) + +# Initialize Flows for individual tasks +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the Sequential Workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) +workflow.add("Summarize the generated blog", flow2) + +# Update the arguments of a specific task in the workflow +workflow.update_task("Generate a 10,000 word blog on health and wellness.", max_loops=2) + +# Output the list of tasks in the workflow after updating task arguments +print("Tasks in the workflow after updating task arguments:") +for task in workflow.tasks: + print(f"Task: {task.description}, Arguments: { + +task.arguments}") +``` + +In this example, we create a Sequential Workflow, add two tasks to it, and then update the arguments of a specific task in the workflow. + +These examples demonstrate various operations and use cases for working with a Sequential Workflow. + +# Why `SequentialWorkflow`? + +## Enhancing Autonomous Agent Development + +The development of autonomous agents, whether they are conversational AI, robotic systems, or any other AI-driven application, often involves complex workflows that require a sequence of tasks to be executed in a specific order. Managing and orchestrating these tasks efficiently is crucial for building reliable and effective agents. The Sequential Workflow module serves as a valuable tool for AI engineers in achieving this goal. + +## Reliability and Coordination + +One of the primary challenges in autonomous agent development is ensuring that tasks are executed in the correct sequence and that the results of one task can be used as inputs for subsequent tasks. The Sequential Workflow module simplifies this process by allowing AI engineers to define and manage workflows in a structured and organized manner. + +By using the Sequential Workflow module, AI engineers can achieve the following benefits: + +### 1. Improved Reliability + +Reliability is a critical aspect of autonomous agents. The ability to handle errors gracefully and recover from failures is essential for building robust systems. The Sequential Workflow module offers a systematic approach to task execution, making it easier to handle errors, retry failed tasks, and ensure that the agent continues to operate smoothly. + +### 2. Task Coordination + +Coordinating tasks in the correct order is essential for achieving the desired outcome. The Sequential Workflow module enforces task sequencing, ensuring that each task is executed only when its dependencies are satisfied. This eliminates the risk of executing tasks out of order, which can lead to incorrect results. + +### 3. Code Organization + +Managing complex workflows can become challenging without proper organization. The Sequential Workflow module encourages AI engineers to structure their code in a modular and maintainable way. Each task can be encapsulated as a separate unit, making it easier to understand, modify, and extend the agent's behavior. + +### 4. Workflow Visualization + +Visualization is a powerful tool for understanding and debugging workflows. The Sequential Workflow module can be extended to include a visualization dashboard, allowing AI engineers to monitor the progress of tasks, track results, and identify bottlenecks or performance issues. + +## TODO: Future Features + +While the Sequential Workflow module offers significant advantages, there are opportunities for further enhancement. Here is a list of potential features and improvements that can be added to make it even more versatile and adaptable for various AI engineering tasks: + +### 1. Asynchronous Support + +Adding support for asynchronous task execution can improve the efficiency of workflows, especially when dealing with tasks that involve waiting for external events or resources. + +### 2. Context Managers + +Introducing context manager support for tasks can simplify resource management, such as opening and closing files, database connections, or network connections within a task's context. + +### 3. Workflow History + +Maintaining a detailed history of workflow execution, including timestamps, task durations, and input/output data, can facilitate debugging and performance analysis. + +### 4. Parallel Processing + +Enhancing the module to support parallel processing with a pool of workers can significantly speed up the execution of tasks, especially for computationally intensive workflows. + +### 5. Error Handling Strategies + +Providing built-in error handling strategies, such as retries, fallbacks, and custom error handling functions, can make the module more robust in handling unexpected failures. + +## Conclusion + +The Sequential Workflow module is a valuable tool for AI engineers working on autonomous agents and complex AI-driven applications. It offers a structured and reliable approach to defining and executing workflows, ensuring that tasks are performed in the correct sequence. By using this module, AI engineers can enhance the reliability, coordination, and maintainability of their agents. + +As the field of AI continues to evolve, the demand for efficient workflow management tools will only increase. The Sequential Workflow module is a step towards meeting these demands and empowering AI engineers to create more reliable and capable autonomous agents. With future enhancements and features, it has the potential to become an indispensable asset in the AI engineer's toolkit. + +In summary, the Sequential Workflow module provides a foundation for orchestrating complex tasks and workflows, enabling AI engineers to focus on designing intelligent agents that can perform tasks with precision and reliability. \ No newline at end of file diff --git a/example.py b/example.py index 3af9fc57..8e34cce3 100644 --- a/example.py +++ b/example.py @@ -1,7 +1,7 @@ from swarms.models import OpenAIChat from swarms.structs import Flow -api_key = "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" +api_key = "" # Initialize the language model, this model can be swapped out with Anthropic, ETC, Huggingface Models like Mistral, ETC llm = OpenAIChat( diff --git a/sequential_workflow_example.py b/sequential_workflow_example.py new file mode 100644 index 00000000..b9ab8196 --- /dev/null +++ b/sequential_workflow_example.py @@ -0,0 +1,37 @@ +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Example usage +api_key = ( + "" # Your actual API key here +) + +# Initialize the language flow +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) + +# Initialize the Flow with the language flow +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create another Flow for a different task +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) + +# Suppose the next task takes the output of the first task as input +workflow.add("Summarize the generated blog", flow2) + +# Run the workflow +workflow.run() + +# Output the results +for task in workflow.tasks: + print(f"Task: {task.description}, Result: {task.result}") diff --git a/swarms/structs/flow.py b/swarms/structs/flow.py index d40e4fb4..0f129314 100644 --- a/swarms/structs/flow.py +++ b/swarms/structs/flow.py @@ -1,8 +1,14 @@ """ TODO: +- add a method that scrapes all the methods from the llm object and outputs them as a string - Add tools - Add open interpreter style conversation - Add memory vector database retrieval +- add batch processing +- add async processing for run and batch run +- add plan module +- concurrent +- """ import json @@ -14,8 +20,15 @@ import inspect import random +# Prompts +DYNAMIC_STOP_PROMPT = """ +When you have finished the task from the Human, output a special token: +This will enable you to leave the autonomous loop. +""" + + # Constants -FLOW_SYSTEM_PROMPT = """ +FLOW_SYSTEM_PROMPT = f""" You are an autonomous agent granted autonomy from a Flow structure. Your role is to engage in multi-step conversations with your self or the user, generate long-form content like blogs, screenplays, or SOPs, @@ -23,19 +36,15 @@ and accomplish tasks. You can have internal dialogues with yourself or can inter to aid in these complex tasks. Your responses should be coherent, contextually relevant, and tailored to the task at hand. -When you have finished the task, and you feel as if you are done: output a special token: -This will enable you to leave the flow loop. +{DYNAMIC_STOP_PROMPT} """ -DYNAMIC_STOP_PROMPT = """ -When you have finished the task, and you feel as if you are done: output a special token: -This will enable you to leave the flow loop. -""" +# Utility functions -# Custome stopping condition +# Custom stopping condition def stop_when_repeats(response: str) -> bool: # Stop if the word stop appears in the response return "Stop" in response.lower() @@ -182,6 +191,7 @@ class Flow: def print_dashboard(self, task: str): """Print dashboard""" model_config = self.get_llm_init_params() + print(colored("Initializing Agent Dashboard...", "yellow")) dashboard = print( colored( @@ -195,6 +205,8 @@ class Flow: ---------------------------------------- Flow Configuration: + Name: {self.name} + System Prompt: {self.system_message} Task: {task} Max Loops: {self.max_loops} Stopping Condition: {self.stopping_condition} @@ -202,14 +214,35 @@ class Flow: Retry Attempts: {self.retry_attempts} Retry Interval: {self.retry_interval} Interactive: {self.interactive} - + Dashboard: {self.dashboard} + Dynamic Temperature: {self.dynamic_temperature} + Autosave: {self.autosave} + Saved State: {self.saved_state} + ---------------------------------------- """, "green", ) ) - print(dashboard) + # print(dashboard) + + def activate_autonomous_agent(self): + """Print the autonomous agent activation message""" + try: + print(colored("Initializing Autonomous Agent...", "yellow")) + # print(colored("Loading modules...", "yellow")) + # print(colored("Modules loaded successfully.", "green")) + print(colored("Autonomous Agent Activated.", "cyan", attrs=["bold"])) + print(colored("All systems operational. Executing task...", "green")) + except Exception as error: + print( + colored( + "Error activating autonomous agent. Try optimizing your parameters...", + "red", + ) + ) + print(error) def run(self, task: str, **kwargs): """ @@ -235,6 +268,11 @@ class Flow: # history = [f"Human: {task}"] # self.memory.append(history) + # print(colored(">>> Autonomous Agent Activated", "cyan", attrs=["bold"])) + self.activate_autonomous_agent() + + # if self.autosave: + response = task history = [f"Human: {task}"] @@ -284,7 +322,10 @@ class Flow: return response # , history - def __call__(self, task: str, save: bool = True, **kwargs): + async def arun(self, task: str, **kwargs): + """Async run""" + pass + """ Run the autonomous agent loop @@ -298,15 +339,17 @@ class Flow: 4. If stopping condition is not met, generate a response 5. Repeat until stopping condition is met or max_loops is reached - Example: - >>> out = flow.run("Generate a 10,000 word blog on health and wellness.") - """ - # Start with a new history or continue from the last saved state - if not self.memory or not self.memory[-1]: - history = [f"Human: {task}"] - else: - history = self.memory[-1] + # Restore from saved state if provided, ortherwise start with a new history + # if self.saved_state: + # self.load_state(self.saved_state) + # history = self.memory[-1] + # print(f"Loaded state from {self.saved_state}") + # else: + # history = [f"Human: {task}"] + # self.memory.append(history) + + print(colored(">>> Autonomous Agent Activated", "cyan", attrs=["bold"])) response = task history = [f"Human: {task}"] @@ -315,12 +358,9 @@ class Flow: if self.dashboard: self.print_dashboard(task) - # Start or continue the loop process - for i in range(len(history), self.max_loops): + for i in range(self.max_loops): print(colored(f"\nLoop {i+1} of {self.max_loops}", "blue")) print("\n") - response = history[-1].split(": ", 1)[-1] # Get the last response - if self._check_stopping_condition(response) or parse_done_token(response): break @@ -332,8 +372,8 @@ class Flow: while attempt < self.retry_attempts: try: response = self.llm( - self.agent_history_prompt(FLOW_SYSTEM_PROMPT, response) - ** kwargs, + self.agent_history_prompt(FLOW_SYSTEM_PROMPT, response), + **kwargs, ) # print(f"Next query: {response}") # break @@ -355,8 +395,8 @@ class Flow: time.sleep(self.loop_interval) self.memory.append(history) - # if save: - # self.save_state("flow_history.json") + # if self.autosave: + # self.save_state("flow_state.json") return response # , history diff --git a/swarms/structs/sequential_workflow.py b/swarms/structs/sequential_workflow.py index f27f3989..c89175f2 100644 --- a/swarms/structs/sequential_workflow.py +++ b/swarms/structs/sequential_workflow.py @@ -1,99 +1,398 @@ """ -Sequential Workflow +TODO: +- Add a method to update the arguments of a task +- Add a method to get the results of each task +- Add a method to get the results of a specific task +- Add a method to get the results of the workflow +- Add a method to get the results of the workflow as a dataframe -from swarms.models import OpenAIChat, Mistral -from swarms.structs import SequentialWorkflow +- Add a method to run the workflow in parallel with a pool of workers and a queue and a dashboard +- Add a dashboard to visualize the workflow +- Add async support +- Add context manager support +- Add workflow history +""" +import json +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Union +from termcolor import colored +from pydantic import BaseModel, validator -llm = OpenAIChat(openai_api_key="") -mistral = Mistral() +from swarms.structs.flow import Flow -# Max loops will run over the sequential pipeline twice -workflow = SequentialWorkflow(max_loops=2) -workflow.add("What's the weather in miami", llm) +# Define a generic Task that can handle different types of callable objects +@dataclass +class Task: + """ + Task class for running a task in a sequential workflow. -workflow.add("Create a report on these metrics", mistral) -workflow.run() + Examples: + >>> from swarms.structs import Task, Flow + >>> from swarms.models import OpenAIChat + >>> flow = Flow(llm=OpenAIChat(openai_api_key=""), max_loops=1, dashboard=False) + >>> task = Task(description="What's the weather in miami", flow=flow) + >>> task.execute() + >>> task.result -""" -from dataclasses import dataclass, field -from typing import List, Any, Dict, Callable, Union -from swarms.models import OpenAIChat -from swarms.structs import Flow -# Define a generic Task that can handle different types of callable objects -@dataclass -class Task: + """ + description: str - model: Union[Callable, Flow] + flow: Union[Callable, Flow] args: List[Any] = field(default_factory=list) kwargs: Dict[str, Any] = field(default_factory=dict) result: Any = None + history: List[Any] = field(default_factory=list) def execute(self): - if isinstance(self.model, Flow): - self.result = self.model.run(*self.args, **self.kwargs) + """ + Execute the task. + + Raises: + ValueError: If a Flow instance is used as a task and the 'task' argument is not provided. + + + + """ + if isinstance(self.flow, Flow): + # Add a prompt to notify the Flow of the sequential workflow + if "prompt" in self.kwargs: + self.kwargs["prompt"] += ( + f"\n\nPrevious output: {self.result}" if self.result else "" + ) + else: + self.kwargs["prompt"] = f"Main task: {self.description}" + ( + f"\n\nPrevious output: {self.result}" if self.result else "" + ) + self.result = self.flow.run(*self.args, **self.kwargs) else: - self.result = self.model(*self.args, **self.kwargs) + self.result = self.flow(*self.args, **self.kwargs) + + self.history.append(self.result) # SequentialWorkflow class definition using dataclasses @dataclass class SequentialWorkflow: + """ + SequentialWorkflow class for running a sequence of tasks using N number of autonomous agents. + + Args: + max_loops (int): The maximum number of times to run the workflow. + dashboard (bool): Whether to display the dashboard for the workflow. + + + Attributes: + tasks (List[Task]): The list of tasks to execute. + max_loops (int): The maximum number of times to run the workflow. + dashboard (bool): Whether to display the dashboard for the workflow. + + + Examples: + >>> from swarms.models import OpenAIChat + >>> from swarms.structs import SequentialWorkflow + >>> llm = OpenAIChat(openai_api_key="") + >>> workflow = SequentialWorkflow(max_loops=1) + >>> workflow.add("What's the weather in miami", llm) + >>> workflow.add("Create a report on these metrics", llm) + >>> workflow.run() + >>> workflow.tasks + + """ + tasks: List[Task] = field(default_factory=list) max_loops: int = 1 + autosave: bool = False + saved_state_filepath: Optional[str] = "sequential_workflow_state.json" + restore_state_filepath: Optional[str] = None + dashboard: bool = False - def add( - self, description: str, model: Union[Callable, Flow], *args, **kwargs - ) -> None: + def add(self, task: str, flow: Union[Callable, Flow], *args, **kwargs) -> None: + """ + Add a task to the workflow. + + Args: + task (str): The task description or the initial input for the Flow. + flow (Union[Callable, Flow]): The model or flow to execute the task. + *args: Additional arguments to pass to the task execution. + **kwargs: Additional keyword arguments to pass to the task execution. + """ + # If the flow is a Flow instance, we include the task in kwargs for Flow.run() + if isinstance(flow, Flow): + kwargs["task"] = task # Set the task as a keyword argument for Flow + + # Append the task to the tasks list self.tasks.append( - Task(description=description, model=model, args=list(args), kwargs=kwargs) + Task(description=task, flow=flow, args=list(args), kwargs=kwargs) ) - def run(self) -> None: - for _ in range(self.max_loops): - for task in self.tasks: - # Check if the current task can be executed - if task.result is None: - task.execute() - # Pass the result as an argument to the next task if it exists - next_task_index = self.tasks.index(task) + 1 - if next_task_index < len(self.tasks): - next_task = self.tasks[next_task_index] - next_task.args.insert(0, task.result) + def reset_workflow(self) -> None: + """Resets the workflow by clearing the results of each task.""" + for task in self.tasks: + task.result = None + def get_task_results(self) -> Dict[str, Any]: + """ + Returns the results of each task in the workflow. -# Example usage -api_key = "" # Your actual API key here + Returns: + Dict[str, Any]: The results of each task in the workflow + """ + return {task.description: task.result for task in self.tasks} -# Initialize the language model -llm = OpenAIChat( - openai_api_key=api_key, - temperature=0.5, - max_tokens=3000, -) + def remove_task(self, task_description: str) -> None: + self.tasks = [ + task for task in self.tasks if task.description != task_description + ] -# Initialize the Flow with the language model -flow1 = Flow(llm=llm, max_loops=5, dashboard=True) + def update_task(self, task_description: str, **updates) -> None: + """ + Updates the arguments of a task in the workflow. -# Create another Flow for a different task -flow2 = Flow(llm=llm, max_loops=5, dashboard=True) + Args: + task_description (str): The description of the task to update. + **updates: The updates to apply to the task. -# Create the workflow -workflow = SequentialWorkflow(max_loops=1) + Raises: + ValueError: If the task is not found in the workflow. -# Add tasks to the workflow -workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) + Examples: + >>> from swarms.models import OpenAIChat + >>> from swarms.structs import SequentialWorkflow + >>> llm = OpenAIChat(openai_api_key="") + >>> workflow = SequentialWorkflow(max_loops=1) + >>> workflow.add("What's the weather in miami", llm) + >>> workflow.add("Create a report on these metrics", llm) + >>> workflow.update_task("What's the weather in miami", max_tokens=1000) + >>> workflow.tasks[0].kwargs + {'max_tokens': 1000} -# Suppose the next task takes the output of the first task as input -workflow.add("Summarize the generated blog", flow2) + """ + for task in self.tasks: + if task.description == task_description: + task.kwargs.update(updates) + break + else: + raise ValueError(f"Task {task_description} not found in workflow.") -# Run the workflow -workflow.run() + def save_workflow_state( + self, filepath: Optional[str] = "sequential_workflow_state.json", **kwargs + ) -> None: + """ + Saves the workflow state to a json file. + + Args: + filepath (str): The path to save the workflow state to. + + Examples: + >>> from swarms.models import OpenAIChat + >>> from swarms.structs import SequentialWorkflow + >>> llm = OpenAIChat(openai_api_key="") + >>> workflow = SequentialWorkflow(max_loops=1) + >>> workflow.add("What's the weather in miami", llm) + >>> workflow.add("Create a report on these metrics", llm) + >>> workflow.save_workflow_state("sequential_workflow_state.json") + """ + filepath = filepath or self.saved_state_filepath + + with open(filepath, "w") as f: + # Saving the state as a json for simplicuty + state = { + "tasks": [ + { + "description": task.description, + "args": task.args, + "kwargs": task.kwargs, + "result": task.result, + "history": task.history, + } + for task in self.tasks + ], + "max_loops": self.max_loops, + } + json.dump(state, f, indent=4) + + def workflow_bootup(self, **kwargs) -> None: + bootup = print( + colored( + f""" + Sequential Workflow Initializing...""", + "green", + attrs=["bold", "underline"], + ) + ) + + def workflow_dashboard(self, **kwargs) -> None: + """ + Displays a dashboard for the workflow. + + Args: + **kwargs: Additional keyword arguments to pass to the dashboard. + + Examples: + >>> from swarms.models import OpenAIChat + >>> from swarms.structs import SequentialWorkflow + >>> llm = OpenAIChat(openai_api_key="") + >>> workflow = SequentialWorkflow(max_loops=1) + >>> workflow.add("What's the weather in miami", llm) + >>> workflow.add("Create a report on these metrics", llm) + >>> workflow.workflow_dashboard() + + """ + dashboard = print( + colored( + f""" + Sequential Workflow Dashboard + -------------------------------- + Tasks: {len(self.tasks)} + Max Loops: {self.max_loops} + Autosave: {self.autosave} + Autosave Filepath: {self.saved_state_filepath} + Restore Filepath: {self.restore_state_filepath} + -------------------------------- + Metadata: + kwargs: {kwargs} + + + + + """, + "cyan", + attrs=["bold", "underline"], + ) + ) + + def load_workflow_state(self, filepath: str = None, **kwargs) -> None: + """ + Loads the workflow state from a json file and restores the workflow state. + + Args: + filepath (str): The path to load the workflow state from. + + Examples: + >>> from swarms.models import OpenAIChat + >>> from swarms.structs import SequentialWorkflow + >>> llm = OpenAIChat(openai_api_key="") + >>> workflow = SequentialWorkflow(max_loops=1) + >>> workflow.add("What's the weather in miami", llm) + >>> workflow.add("Create a report on these metrics", llm) + >>> workflow.save_workflow_state("sequential_workflow_state.json") + >>> workflow.load_workflow_state("sequential_workflow_state.json") + + """ + filepath = filepath or self.restore_state_filepath + + with open(filepath, "r") as f: + state = json.load(f) + self.max_loops = state["max_loops"] + self.tasks = [] + for task_state in state["tasks"]: + task = Task( + description=task_state["description"], + flow=task_state["flow"], + args=task_state["args"], + kwargs=task_state["kwargs"], + result=task_state["result"], + history=task_state["history"], + ) + self.tasks.append(task) + + def run(self) -> None: + """ + Run the workflow. + + Raises: + ValueError: If a Flow instance is used as a task and the 'task' argument is not provided. + + """ + try: + self.workflow_bootup() + for _ in range(self.max_loops): + for task in self.tasks: + # Check if the current task can be executed + if task.result is None: + # Check if the flow is a Flow and a 'task' argument is needed + if isinstance(task.flow, Flow): + # Ensure that 'task' is provided in the kwargs + if "task" not in task.kwargs: + raise ValueError( + f"The 'task' argument is required for the Flow flow execution in '{task.description}'" + ) + # Separate the 'task' argument from other kwargs + flow_task_arg = task.kwargs.pop("task") + task.result = task.flow.run( + flow_task_arg, *task.args, **task.kwargs + ) + else: + # If it's not a Flow instance, call the flow directly + task.result = task.flow(*task.args, **task.kwargs) + + # Pass the result as an argument to the next task if it exists + next_task_index = self.tasks.index(task) + 1 + if next_task_index < len(self.tasks): + next_task = self.tasks[next_task_index] + if isinstance(next_task.flow, Flow): + # For Flow flows, 'task' should be a keyword argument + next_task.kwargs["task"] = task.result + else: + # For other callable flows, the result is added to args + next_task.args.insert(0, task.result) + + # Autosave the workflow state + if self.autosave: + self.save_workflow_state("sequential_workflow_state.json") + except Exception as e: + print( + colored( + f"Error initializing the Sequential workflow: {e} try optimizing your inputs like the flow class and task description", + "red", + attrs=["bold", "underline"], + ) + ) + + async def arun(self) -> None: + """ + Asynchronously run the workflow. + + Raises: + ValueError: If a Flow instance is used as a task and the 'task' argument is not provided. + + """ + for _ in range(self.max_loops): + for task in self.tasks: + # Check if the current task can be executed + if task.result is None: + # Check if the flow is a Flow and a 'task' argument is needed + if isinstance(task.flow, Flow): + # Ensure that 'task' is provided in the kwargs + if "task" not in task.kwargs: + raise ValueError( + f"The 'task' argument is required for the Flow flow execution in '{task.description}'" + ) + # Separate the 'task' argument from other kwargs + flow_task_arg = task.kwargs.pop("task") + task.result = await task.flow.arun( + flow_task_arg, *task.args, **task.kwargs + ) + else: + # If it's not a Flow instance, call the flow directly + task.result = await task.flow(*task.args, **task.kwargs) + + # Pass the result as an argument to the next task if it exists + next_task_index = self.tasks.index(task) + 1 + if next_task_index < len(self.tasks): + next_task = self.tasks[next_task_index] + if isinstance(next_task.flow, Flow): + # For Flow flows, 'task' should be a keyword argument + next_task.kwargs["task"] = task.result + else: + # For other callable flows, the result is added to args + next_task.args.insert(0, task.result) -# Output the results -for task in workflow.tasks: - print(f"Task: {task.description}, Result: {task.result}") + # Autosave the workflow state + if self.autosave: + self.save_workflow_state("sequential_workflow_state.json") diff --git a/swarms/swarms/autobloggen.py b/swarms/swarms/autobloggen.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/structs/sequential_workflow.py b/tests/structs/sequential_workflow.py new file mode 100644 index 00000000..64b51f28 --- /dev/null +++ b/tests/structs/sequential_workflow.py @@ -0,0 +1,306 @@ +import asyncio +import os +from unittest.mock import patch + +import pytest + +from swarms.models import OpenAIChat +from swarms.structs.flow import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow, Task + +# Mock the OpenAI API key using environment variables +os.environ["OPENAI_API_KEY"] = "mocked_api_key" + + + +# Mock OpenAIChat class for testing +class MockOpenAIChat: + def __init__(self, *args, **kwargs): + pass + + def run(self, *args, **kwargs): + return "Mocked result" + +# Mock Flow class for testing +class MockFlow: + def __init__(self, *args, **kwargs): + pass + + def run(self, *args, **kwargs): + return "Mocked result" + +# Mock SequentialWorkflow class for testing +class MockSequentialWorkflow: + def __init__(self, *args, **kwargs): + pass + + def add(self, *args, **kwargs): + pass + + def run(self): + pass + +# Test Task class +def test_task_initialization(): + description = "Sample Task" + flow = MockOpenAIChat() + task = Task(description=description, flow=flow) + assert task.description == description + assert task.flow == flow + +def test_task_execute(): + description = "Sample Task" + flow = MockOpenAIChat() + task = Task(description=description, flow=flow) + task.execute() + assert task.result == "Mocked result" + +# Test SequentialWorkflow class +def test_sequential_workflow_initialization(): + workflow = SequentialWorkflow() + assert isinstance(workflow, SequentialWorkflow) + assert len(workflow.tasks) == 0 + assert workflow.max_loops == 1 + assert workflow.autosave == False + assert workflow.saved_state_filepath == "sequential_workflow_state.json" + assert workflow.restore_state_filepath == None + assert workflow.dashboard == False + +def test_sequential_workflow_add_task(): + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = MockOpenAIChat() + workflow.add(task_description, task_flow) + assert len(workflow.tasks) == 1 + assert workflow.tasks[0].description == task_description + assert workflow.tasks[0].flow == task_flow + +def test_sequential_workflow_reset_workflow(): + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = MockOpenAIChat() + workflow.add(task_description, task_flow) + workflow.reset_workflow() + assert workflow.tasks[0].result == None + +def test_sequential_workflow_get_task_results(): + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = MockOpenAIChat() + workflow.add(task_description, task_flow) + workflow.run() + results = workflow.get_task_results() + assert len(results) == 1 + assert task_description in results + assert results[task_description] == "Mocked result" + +def test_sequential_workflow_remove_task(): + workflow = SequentialWorkflow() + task1_description = "Task 1" + task2_description = "Task 2" + task1_flow = MockOpenAIChat() + task2_flow = MockOpenAIChat() + workflow.add(task1_description, task1_flow) + workflow.add(task2_description, task2_flow) + workflow.remove_task(task1_description) + assert len(workflow.tasks) == 1 + assert workflow.tasks[0].description == task2_description + +def test_sequential_workflow_update_task(): + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = MockOpenAIChat() + workflow.add(task_description, task_flow) + workflow.update_task(task_description, max_tokens=1000) + assert workflow.tasks[0].kwargs["max_tokens"] == 1000 + +def test_sequential_workflow_save_workflow_state(): + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = MockOpenAIChat() + workflow.add(task_description, task_flow) + workflow.save_workflow_state("test_state.json") + assert os.path.exists("test_state.json") + os.remove("test_state.json") + +def test_sequential_workflow_load_workflow_state(): + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = MockOpenAIChat() + workflow.add(task_description, task_flow) + workflow.save_workflow_state("test_state.json") + workflow.load_workflow_state("test_state.json") + assert len(workflow.tasks) == 1 + assert workflow.tasks[0].description == task_description + os.remove("test_state.json") + +def test_sequential_workflow_run(): + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = MockOpenAIChat() + workflow.add(task_description, task_flow) + workflow.run() + assert workflow.tasks[0].result == "Mocked result" + +def test_sequential_workflow_workflow_bootup(capfd): + workflow = SequentialWorkflow() + workflow.workflow_bootup() + out, _ = capfd.readouterr() + assert "Sequential Workflow Initializing..." in out + +def test_sequential_workflow_workflow_dashboard(capfd): + workflow = SequentialWorkflow() + workflow.workflow_dashboard() + out, _ = capfd.readouterr() + assert "Sequential Workflow Dashboard" in out + +# Mock Flow class for async testing +class MockAsyncFlow: + def __init__(self, *args, **kwargs): + pass + + async def arun(self, *args, **kwargs): + return "Mocked result" + +# Test async execution in SequentialWorkflow +@pytest.mark.asyncio +async def test_sequential_workflow_arun(): + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = MockAsyncFlow() + workflow.add(task_description, task_flow) + await workflow.arun() + assert workflow.tasks[0].result == "Mocked result" + + + + +def test_real_world_usage_with_openai_key(): + # Initialize the language model + llm = OpenAIChat() + assert isinstance(llm, OpenAIChat) + +def test_real_world_usage_with_flow_and_openai_key(): + # Initialize a flow with the language model + flow = Flow(llm=OpenAIChat()) + assert isinstance(flow, Flow) + +def test_real_world_usage_with_sequential_workflow(): + # Initialize a sequential workflow + workflow = SequentialWorkflow() + assert isinstance(workflow, SequentialWorkflow) + +def test_real_world_usage_add_tasks(): + # Create a sequential workflow and add tasks + workflow = SequentialWorkflow() + task1_description = "Task 1" + task2_description = "Task 2" + task1_flow = OpenAIChat() + task2_flow = OpenAIChat() + workflow.add(task1_description, task1_flow) + workflow.add(task2_description, task2_flow) + assert len(workflow.tasks) == 2 + assert workflow.tasks[0].description == task1_description + assert workflow.tasks[1].description == task2_description + +def test_real_world_usage_run_workflow(): + # Create a sequential workflow, add a task, and run the workflow + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = OpenAIChat() + workflow.add(task_description, task_flow) + workflow.run() + assert workflow.tasks[0].result is not None + +def test_real_world_usage_dashboard_display(): + # Create a sequential workflow, add tasks, and display the dashboard + workflow = SequentialWorkflow() + task1_description = "Task 1" + task2_description = "Task 2" + task1_flow = OpenAIChat() + task2_flow = OpenAIChat() + workflow.add(task1_description, task1_flow) + workflow.add(task2_description, task2_flow) + with patch("builtins.print") as mock_print: + workflow.workflow_dashboard() + mock_print.assert_called() + +def test_real_world_usage_async_execution(): + # Create a sequential workflow, add an async task, and run the workflow asynchronously + workflow = SequentialWorkflow() + task_description = "Sample Task" + async_task_flow = OpenAIChat() + + async def async_run_workflow(): + await workflow.arun() + + workflow.add(task_description, async_task_flow) + asyncio.run(async_run_workflow()) + assert workflow.tasks[0].result is not None + +def test_real_world_usage_multiple_loops(): + # Create a sequential workflow with multiple loops, add a task, and run the workflow + workflow = SequentialWorkflow(max_loops=3) + task_description = "Sample Task" + task_flow = OpenAIChat() + workflow.add(task_description, task_flow) + workflow.run() + assert workflow.tasks[0].result is not None + +def test_real_world_usage_autosave_state(): + # Create a sequential workflow with autosave, add a task, run the workflow, and check if state is saved + workflow = SequentialWorkflow(autosave=True) + task_description = "Sample Task" + task_flow = OpenAIChat() + workflow.add(task_description, task_flow) + workflow.run() + assert workflow.tasks[0].result is not None + assert os.path.exists("sequential_workflow_state.json") + os.remove("sequential_workflow_state.json") + +def test_real_world_usage_load_state(): + # Create a sequential workflow, add a task, save state, load state, and run the workflow + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = OpenAIChat() + workflow.add(task_description, task_flow) + workflow.run() + workflow.save_workflow_state("test_state.json") + workflow.load_workflow_state("test_state.json") + workflow.run() + assert workflow.tasks[0].result is not None + os.remove("test_state.json") + +def test_real_world_usage_update_task_args(): + # Create a sequential workflow, add a task, and update task arguments + workflow = SequentialWorkflow() + task_description = "Sample Task" + task_flow = OpenAIChat() + workflow.add(task_description, task_flow) + workflow.update_task(task_description, max_tokens=1000) + assert workflow.tasks[0].kwargs["max_tokens"] == 1000 + +def test_real_world_usage_remove_task(): + # Create a sequential workflow, add tasks, remove a task, and run the workflow + workflow = SequentialWorkflow() + task1_description = "Task 1" + task2_description = "Task 2" + task1_flow = OpenAIChat() + task2_flow = OpenAIChat() + workflow.add(task1_description, task1_flow) + workflow.add(task2_description, task2_flow) + workflow.remove_task(task1_description) + workflow.run() + assert len(workflow.tasks) == 1 + assert workflow.tasks[0].description == task2_description + +def test_real_world_usage_with_environment_variables(): + # Ensure that the OpenAI API key is set using environment variables + assert "OPENAI_API_KEY" in os.environ + assert os.environ["OPENAI_API_KEY"] == "mocked_api_key" + del os.environ["OPENAI_API_KEY"] # Clean up after the test + +def test_real_world_usage_no_openai_key(): + # Ensure that an exception is raised when the OpenAI API key is not set + with pytest.raises(ValueError): + llm = OpenAIChat() # API key not provided, should raise an exception \ No newline at end of file From c94512d6548c2076d7bf4cdb67b59f4a87c54b67 Mon Sep 17 00:00:00 2001 From: Kye Date: Sun, 5 Nov 2023 23:26:59 -0500 Subject: [PATCH 14/32] sequential workflow docs --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 55c7cf3d..abd2bd42 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -111,6 +111,7 @@ nav: - Overview: "swarms/structs/overview.md" - Workflow: "swarms/structs/workflow.md" - Flow: "swarms/structs/flow.md" + - SequentialWorkflow: 'swarms/structs/sequential_workflow.md' - swarms.memory: - PineconeVectorStoreStore: "swarms/memory/pinecone.md" - PGVectorStore: "swarms/memory/pg.md" From 383412bace9c4a0bafefdefdcc528b0620ff6c38 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 08:35:45 -0500 Subject: [PATCH 15/32] workflow states --- README.md | 45 ++++++++++++++++++++++ docs/swarms/structs/sequential_workflow.md | 39 ++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 68d7ba05..f94221d4 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,51 @@ god_mode.print_responses(task) ------ +### `SequentialWorkflow` +- Execute tasks step by step by passing in an LLM and the task description! +- Pass in flows with various LLMs +- Save and restore Workflow states! +```python +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Example usage +api_key = ( + "" # Your actual API key here +) + +# Initialize the language flow +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) + +# Initialize the Flow with the language flow +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create another Flow for a different task +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) + +# Suppose the next task takes the output of the first task as input +workflow.add("Summarize the generated blog", flow2) + +# Run the workflow +workflow.run() + +# Output the results +for task in workflow.tasks: + print(f"Task: {task.description}, Result: {task.result}") + +``` + ### `OmniModalAgent` - OmniModal Agent is an LLM that access to 10+ multi-modal encoders and diffusers! It can generate images, videos, speech, music and so much more, get started with: diff --git a/docs/swarms/structs/sequential_workflow.md b/docs/swarms/structs/sequential_workflow.md index 04587b89..12b38409 100644 --- a/docs/swarms/structs/sequential_workflow.md +++ b/docs/swarms/structs/sequential_workflow.md @@ -574,4 +574,41 @@ The Sequential Workflow module is a valuable tool for AI engineers working on au As the field of AI continues to evolve, the demand for efficient workflow management tools will only increase. The Sequential Workflow module is a step towards meeting these demands and empowering AI engineers to create more reliable and capable autonomous agents. With future enhancements and features, it has the potential to become an indispensable asset in the AI engineer's toolkit. -In summary, the Sequential Workflow module provides a foundation for orchestrating complex tasks and workflows, enabling AI engineers to focus on designing intelligent agents that can perform tasks with precision and reliability. \ No newline at end of file +In summary, the Sequential Workflow module provides a foundation for orchestrating complex tasks and workflows, enabling AI engineers to focus on designing intelligent agents that can perform tasks with precision and reliability. + + +## Frequently Asked Questions (FAQs) + +### Q1: What is the difference between a task and a flow in Sequential Workflows? + +**A1:** In Sequential Workflows, a **task** refers to a specific unit of work that needs to be executed. It can be implemented as a callable object, such as a Python function, and is the fundamental building block of a workflow. + +A **flow**, on the other hand, is an encapsulation of a task within the workflow. Flows define the order in which tasks are executed and can be thought of as task containers. They allow you to specify dependencies, error handling, and other workflow-related configurations. + +### Q2: Can I run tasks in parallel within a Sequential Workflow? + +**A2:** Yes, you can run tasks in parallel within a Sequential Workflow by using parallel execution techniques. This advanced feature allows you to execute multiple tasks concurrently, improving performance and efficiency. You can explore this feature further in the guide's section on "Parallel Execution." + +### Q3: How do I handle errors within Sequential Workflows? + +**A3:** Error handling within Sequential Workflows can be implemented by adding error-handling logic within your task functions. You can catch exceptions and handle errors gracefully, ensuring that your workflow can recover from unexpected scenarios. The guide also covers more advanced error handling strategies, such as retrying failed tasks and handling specific error types. + +### Q4: What are some real-world use cases for Sequential Workflows? + +**A4:** Sequential Workflows can be applied to a wide range of real-world use cases, including: + +- **Data ETL (Extract, Transform, Load) Processes:** Automating data pipelines that involve data extraction, transformation, and loading into databases or data warehouses. + +- **Batch Processing:** Running batch jobs that process large volumes of data or perform data analysis. + +- **Automation of DevOps Tasks:** Streamlining DevOps processes such as deployment, provisioning, and monitoring. + +- **Cross-system Integrations:** Automating interactions between different systems, services, or APIs. + +- **Report Generation:** Generating reports and documents automatically based on data inputs. + +- **Workflow Orchestration:** Orchestrating complex workflows involving multiple steps and dependencies. + +- **Resource Provisioning:** Automatically provisioning and managing cloud resources. + +These are just a few examples, and Sequential Workflows can be tailored to various automation needs across industries. From 71da697cc8dedac243bd73b9b6c8b09def6e3717 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 11:36:08 -0500 Subject: [PATCH 16/32] sequential workflow --- docs/examples/reliable_autonomous_agents.md | 229 ++++++++++++++++++++ pyproject.toml | 2 +- swarms/structs/__init__.py | 3 +- swarms/structs/sequential_workflow.py | 8 +- 4 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 docs/examples/reliable_autonomous_agents.md diff --git a/docs/examples/reliable_autonomous_agents.md b/docs/examples/reliable_autonomous_agents.md new file mode 100644 index 00000000..21d0478b --- /dev/null +++ b/docs/examples/reliable_autonomous_agents.md @@ -0,0 +1,229 @@ +# Reliable Enterprise-Grade Autonomous Agents in Less Than 5 lines of Code +======================================================================== + +Welcome to this comprehensive walkthrough guide tutorial on the SequentialWorkflow feature of the Swarms Framework! In this tutorial, we will explore the purpose, usage, and key concepts of the SequentialWorkflow class, which is a part of the swarms package. Whether you are a beginner, intermediate, or expert developer, this tutorial will provide you with a clear understanding of how to effectively use the SequentialWorkflow class in your projects. + +AI engineering is a dynamic and evolving field that involves the development and deployment of intelligent systems and applications. In this ever-changing landscape, AI engineers often face the challenge of orchestrating complex sequences of tasks, managing data flows, and ensuring the smooth execution of AI workflows. This is where the Workflow Class, such as the SequentialWorkflow class we discussed earlier, plays a pivotal role in enabling AI engineers to achieve their goals efficiently and effectively. + +The Versatile World of AI Workflows +AI workflows encompass a wide range of tasks and processes, from data preprocessing and model training to natural language understanding and decision-making. These workflows are the backbone of AI systems, guiding them through intricate sequences of actions to deliver meaningful results. Here are some of the diverse use cases where the Workflow Class can empower AI engineers: + +1. Natural Language Processing (NLP) Pipelines +AI engineers often build NLP pipelines that involve multiple stages such as text preprocessing, tokenization, feature extraction, model inference, and post-processing. The Workflow Class enables the orderly execution of these stages, ensuring that textual data flows seamlessly through each step, resulting in accurate and coherent NLP outcomes. + +2. Data Ingestion and Transformation +AI projects frequently require the ingestion of diverse data sources, including structured databases, unstructured text, and multimedia content. The Workflow Class can be used to design data ingestion workflows that extract, transform, and load (ETL) data efficiently, making it ready for downstream AI tasks like training and analysis. + +3. Autonomous Agents and Robotics +In autonomous robotics and intelligent agent systems, workflows are essential for decision-making, sensor fusion, motion planning, and control. AI engineers can use the Workflow Class to create structured sequences of actions that guide robots and agents through dynamic environments, enabling them to make informed decisions and accomplish tasks autonomously. + +4. Machine Learning Model Training +Training machine learning models involves a series of steps, including data preprocessing, feature engineering, model selection, hyperparameter tuning, and evaluation. The Workflow Class simplifies the orchestration of these steps, allowing AI engineers to experiment with different configurations and track the progress of model training. + +5. Content Generation and Summarization +AI-driven content generation tasks, such as generating articles, reports, or summaries, often require multiple steps, including content creation and post-processing. The Workflow Class can be used to create content generation workflows, ensuring that the generated content meets quality and coherence criteria. + +6. Adaptive Decision-Making +In AI systems that make real-time decisions based on changing data and environments, workflows facilitate adaptive decision-making. Engineers can use the Workflow Class to design decision-making pipelines that take into account the latest information and make informed choices. + +Enabling Efficiency and Maintainability +The Workflow Class provides AI engineers with a structured and maintainable approach to building, executing, and managing complex AI workflows. It offers the following advantages: + +Modularity: Workflows can be modularly designed, allowing engineers to focus on individual task implementations and ensuring code reusability. + +Debugging and Testing: The Workflow Class simplifies debugging and testing by providing a clear sequence of tasks and well-defined inputs and outputs for each task. + +Scalability: As AI projects grow in complexity, the Workflow Class can help manage and scale workflows by adding or modifying tasks as needed. + +Error Handling: The class supports error handling strategies, enabling engineers to define how to handle unexpected failures gracefully. + +Maintainability: With structured workflows, AI engineers can easily maintain and update AI systems as requirements evolve or new data sources become available. + +The Workflow Class, such as the SequentialWorkflow class, is an indispensable tool in the toolkit of AI engineers. It empowers engineers to design, execute, and manage AI workflows across a diverse range of use cases. By providing structure, modularity, and maintainability to AI projects, the Workflow Class contributes significantly to the efficiency and success of AI engineering endeavors. As the field of AI continues to advance, harnessing the power of workflow orchestration will remain a key ingredient in building intelligent and adaptable systems, now let’s get started with SequentialWorkflow. + +## Official Swarms Links +Here is the Swarms website: + +Here is the Swarms Github: + +Here are the Swarms docs: + +And, join the Swarm community! + +Book a call with The Swarm Corporation here if you’re interested in high performance custom swarms! + +Now let’s begin… + +## Installation +Before we dive into the tutorial, make sure you have the following prerequisites in place: + +Python installed on your system. +The swarms library installed. You can install it via pip using the following command: + +`pip3 install --upgrade swarms` + +Additionally, you will need an API key for the OpenAIChat model to run the provided code examples. Replace "YOUR_API_KEY" with your actual API key in the code examples where applicable. + +## Getting Started +Let’s start by importing the necessary modules and initializing the OpenAIChat model, which we will use in our workflow tasks. + + +```python +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Replace "YOUR_API_KEY" with your actual OpenAI API key +api_key = "YOUR_API_KEY" + +# Initialize the language model flow (e.g., GPT-3) +llm = OpenAIChat( + openai_api_key=api_key, + temperature=0.5, + max_tokens=3000, +) +We have initialized the OpenAIChat model, which will be used as a callable object in our tasks. Now, let’s proceed to create the SequentialWorkflow. + +Creating a SequentialWorkflow +To create a SequentialWorkflow, follow these steps: + +# Initialize Flows for individual tasks +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) +# Create the Sequential Workflow +workflow = SequentialWorkflow(max_loops=1) +`````` +In this code snippet, we have initialized two Flow instances (flow1 and flow2) representing individual tasks within our workflow. These flows will use the OpenAIChat model we initialized earlier. We then create a SequentialWorkflow instance named workflow with a maximum loop count of 1. The max_loops parameter determines how many times the entire workflow can be run, and we set it to 1 for this example. + +Adding Tasks to the SequentialWorkflow +Now that we have created the SequentialWorkflow, let’s add tasks to it. In our example, we’ll create two tasks: one for generating a 10,000-word blog on “health and wellness” and another for summarizing the generated blog. + +``` +### Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) + +`workflow.add("Summarize the generated blog", flow2)` + +The workflow.add() method is used to add tasks to the workflow. Each task is described using a human-readable description, such as "Generate a 10,000 word blog on health and wellness," and is associated with a flow (callable object) that will be executed as the task. In our example, flow1 and flow2 represent the tasks. + +Running the SequentialWorkflow +With tasks added to the SequentialWorkflow, we can now run the workflow sequentially using the workflow.run() method. + +### Run the workflow +`workflow.run()` +Executing workflow.run() will start the execution of tasks in the order they were added to the workflow. In our example, it will first generate the blog and then summarize it. + +Accessing Task Results +After running the workflow, you can access the results of each task using the get_task_results() method. + +# Get and display the results of each task in the workflow +```python +results = workflow.get_task_results() +for task_description, result in results.items(): + print(f"Task: {task_description}, Result: {result}") +``` +The workflow.get_task_results() method returns a dictionary where the keys are task descriptions, and the values are the corresponding results. You can then iterate through the results and print them, as shown in the code snippet. + +Resetting a SequentialWorkflow +Sometimes, you might need to reset a SequentialWorkflow to start fresh. You can use the workflow.reset_workflow() method for this purpose. + +### Reset the workflow +`workflow.reset_workflow()` +Resetting the workflow clears the results of each task, allowing you to rerun the workflow from the beginning without reinitializing it. + +Updating Task Arguments +You can also update the arguments of a specific task in the workflow using the workflow.update_task() method. + +### Update the arguments of a specific task in the workflow +`workflow.update_task("Generate a 10,000 word blog on health and wellness.", max_loops=2)` + +In this example, we update the max_loops argument of the task with the description "Generate a 10,000 word blog on health and wellness" to 2. This can be useful if you want to change the behavior of a specific task without recreating the entire workflow. + +# Conclusion: Mastering Workflow Orchestration in AI Engineering +In the ever-evolving landscape of artificial intelligence (AI), where the pace of innovation and complexity of tasks are ever-increasing, harnessing the power of workflow orchestration is paramount. In this comprehensive walkthrough guide, we’ve embarked on a journey through the world of workflow orchestration, focusing on the Workflow Class, with a specific emphasis on the SequentialWorkflow class. As we conclude this exploration, we’ve delved deep into the intricacies of orchestrating AI workflows, and it’s time to reflect on the valuable insights gained and the immense potential that this knowledge unlocks for AI engineers. + +The Art of Workflow Orchestration +At its core, workflow orchestration is the art of designing, managing, and executing sequences of tasks or processes in a structured and efficient manner. In the realm of AI engineering, where tasks can range from data preprocessing and model training to decision-making and autonomous actions, mastering workflow orchestration is a game-changer. It empowers AI engineers to streamline their work, ensure reliable execution, and deliver impactful results. + +The Workflow Class, and particularly the SequentialWorkflow class we’ve explored, acts as a guiding light in this intricate journey. It provides AI engineers with a toolbox of tools and techniques to conquer the challenges of orchestrating AI workflows effectively. Through a disciplined approach and adherence to best practices, AI engineers can achieve the following: + +1. Structured Workflow Design +A well-structured workflow is the cornerstone of any successful AI project. The Workflow Class encourages AI engineers to break down complex tasks into manageable units. Each task becomes a building block that contributes to the overarching goal. Whether it’s preprocessing data, training a machine learning model, or generating content, structured workflow design ensures clarity, modularity, and maintainability. + +2. Efficient Task Sequencing +In AI, the order of tasks often matters. One task’s output can be another task’s input, and ensuring the correct sequence of execution is crucial. The SequentialWorkflow class enforces this sequential execution, eliminating the risk of running tasks out of order. It ensures that the workflow progresses systematically, following the predefined sequence of tasks. + +3. Error Resilience and Recovery +AI systems must be resilient in the face of unexpected errors and failures. The Workflow Class equips AI engineers with error handling strategies, such as retries and fallbacks. These strategies provide the ability to gracefully handle issues, recover from failures, and continue the workflow’s execution without disruption. + +4. Code Modularity and Reusability +Building AI workflows often involves implementing various tasks, each with its own logic. The Workflow Class encourages code modularity, allowing AI engineers to encapsulate tasks as separate units. This modularity promotes code reusability, making it easier to adapt and expand workflows as AI projects evolve. + +5. Efficient Debugging and Testing +Debugging and testing AI workflows can be challenging without clear structure and boundaries. The Workflow Class provides a clear sequence of tasks with well-defined inputs and outputs. This structure simplifies the debugging process, as AI engineers can isolate and test individual tasks, ensuring that each component functions as intended. + +6. Scalability and Adaptability +As AI projects grow in complexity, the Workflow Class scales effortlessly. AI engineers can add or modify tasks as needed, accommodating new data sources, algorithms, or requirements. This scalability ensures that workflows remain adaptable to changing demands and evolving AI landscapes. + +7. Maintainability and Future-Proofing +Maintaining AI systems over time is a crucial aspect of engineering. The Workflow Class fosters maintainability by providing a clear roadmap of tasks and their interactions. AI engineers can revisit, update, and extend workflows with confidence, ensuring that AI systems remain effective and relevant in the long run. + +Empowering AI Engineers +The knowledge and skills gained from this walkthrough guide go beyond technical proficiency. They empower AI engineers to be architects of intelligent systems, capable of orchestrating AI workflows that solve real-world problems. The Workflow Class is a versatile instrument in their hands, enabling them to tackle diverse use cases and engineering challenges. + +Diverse Use Cases for Workflow Class +Throughout this guide, we explored a myriad of use cases where the Workflow Class shines: + +Natural Language Processing (NLP) Pipelines: In NLP, workflows involve multiple stages, and the Workflow Class ensures orderly execution, resulting in coherent NLP outcomes. +Data Ingestion and Transformation: Data is the lifeblood of AI, and structured data workflows ensure efficient data preparation for downstream tasks. +Autonomous Agents and Robotics: For robots and intelligent agents, workflows enable autonomous decision-making and task execution. +Machine Learning Model Training: Model training workflows encompass numerous steps, and structured orchestration simplifies the process. +Content Generation and Summarization: Workflows for content generation ensure that generated content meets quality and coherence criteria. +Adaptive Decision-Making: In dynamic environments, workflows facilitate adaptive decision-making based on real-time data. +Efficiency and Maintainability +AI engineers not only have the tools to tackle these use cases but also the means to do so efficiently. The Workflow Class fosters efficiency and maintainability, making AI engineering endeavors more manageable: + +Modularity: Encapsulate tasks as separate units, promoting code reusability and maintainability. +Debugging and Testing: Streamline debugging and testing through clear task boundaries and well-defined inputs and outputs. +Scalability: As AI projects grow, workflows scale with ease, accommodating new components and requirements. +Error Handling: Gracefully handle errors and failures, ensuring that AI systems continue to operate smoothly. +Maintainability: AI systems remain adaptable and maintainable, even as the AI landscape evolves and requirements change. +The Future of AI Engineering +As AI engineering continues to advance, workflow orchestration will play an increasingly pivotal role. The Workflow Class is not a static tool; it is a dynamic enabler of innovation. In the future, we can expect further enhancements and features to meet the evolving demands of AI engineering: + +1. Asynchronous Support +Support for asynchronous task execution will improve the efficiency of workflows, especially when tasks involve waiting for external events or resources. + +2. Context Managers +Introducing context manager support for tasks can simplify resource management, such as opening and closing files or database connections. + +3. Workflow History +Maintaining a detailed history of workflow execution, including timestamps, task durations, and input/output data, will facilitate debugging and performance analysis. + +4. Parallel Processing +Enhancing the module to support parallel processing with a pool of workers can significantly speed up the execution of tasks, especially for computationally intensive workflows. + +5. Error Handling Strategies +Providing built-in error handling strategies, such as retries, fallbacks, and circuit breakers, will further enhance the resilience of workflows. + +Closing Thoughts +In conclusion, the journey through workflow orchestration in AI engineering has been both enlightening and empowering. The Workflow Class, and particularly the SequentialWorkflow class, has proven to be an invaluable ally in the AI engineer’s toolkit. It offers structure, modularity, and efficiency, ensuring that AI projects progress smoothly from inception to deployment. + +As AI continues to permeate every aspect of our lives, the skills acquired in this guide will remain highly relevant and sought after. AI engineers armed with workflow orchestration expertise will continue to push the boundaries of what is possible, solving complex problems, and driving innovation. + +But beyond the technical aspects, this guide also emphasizes the importance of creativity, adaptability, and problem-solving. AI engineering is not just about mastering tools; it’s about using them to make a meaningful impact on the world. + +So, whether you’re just starting your journey into AI engineering or you’re a seasoned professional seeking to expand your horizons, remember that the power of workflow orchestration lies not only in the code but in the limitless potential it unlocks for you as an AI engineer. As you embark on your own AI adventures, may this guide serve as a reliable companion, illuminating your path and inspiring your journey towards AI excellence. + +The world of AI is waiting for your innovation and creativity. With workflow orchestration as your guide, you have the tools to shape the future. The possibilities are boundless, and the future is yours to create. + +Official Swarms Links +Here is the Swarms website: + +Here is the Swarms Github: + +Here are the Swarms docs: + +And, join the Swarm community! + +Book a call with The Swarm Corporation here if you’re interested in high performance custom swarms! \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4af20ee0..e3a29e78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "swarms" -version = "1.9.3" +version = "1.9.5" description = "Swarms - Pytorch" license = "MIT" authors = ["Kye Gomez "] diff --git a/swarms/structs/__init__.py b/swarms/structs/__init__.py index d360fa78..a842359c 100644 --- a/swarms/structs/__init__.py +++ b/swarms/structs/__init__.py @@ -1,5 +1,6 @@ from swarms.structs.workflow import Workflow from swarms.structs.task import Task from swarms.structs.flow import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow -__all__ = ["Workflow", "Task", "Flow"] +__all__ = ["Workflow", "Task", "Flow", "SequentialWorkflow"] diff --git a/swarms/structs/sequential_workflow.py b/swarms/structs/sequential_workflow.py index c89175f2..802e5442 100644 --- a/swarms/structs/sequential_workflow.py +++ b/swarms/structs/sequential_workflow.py @@ -16,8 +16,8 @@ TODO: import json from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Union + from termcolor import colored -from pydantic import BaseModel, validator from swarms.structs.flow import Flow @@ -217,9 +217,9 @@ class SequentialWorkflow: json.dump(state, f, indent=4) def workflow_bootup(self, **kwargs) -> None: - bootup = print( + print( colored( - f""" + """ Sequential Workflow Initializing...""", "green", attrs=["bold", "underline"], @@ -243,7 +243,7 @@ class SequentialWorkflow: >>> workflow.workflow_dashboard() """ - dashboard = print( + print( colored( f""" Sequential Workflow Dashboard From 6010c9d689cbab181c23c3f63ae86d083f8a6731 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 11:59:50 -0500 Subject: [PATCH 17/32] swarms docs corporate + sequential workflow --- docs/examples/reliable_autonomous_agents.md | 82 ++++++++++++--------- mkdocs.yml | 30 ++++---- 2 files changed, 62 insertions(+), 50 deletions(-) diff --git a/docs/examples/reliable_autonomous_agents.md b/docs/examples/reliable_autonomous_agents.md index 21d0478b..f2988075 100644 --- a/docs/examples/reliable_autonomous_agents.md +++ b/docs/examples/reliable_autonomous_agents.md @@ -1,43 +1,43 @@ -# Reliable Enterprise-Grade Autonomous Agents in Less Than 5 lines of Code +# Enterprise-Grade Workflow Automation With Autonomous Agents ======================================================================== Welcome to this comprehensive walkthrough guide tutorial on the SequentialWorkflow feature of the Swarms Framework! In this tutorial, we will explore the purpose, usage, and key concepts of the SequentialWorkflow class, which is a part of the swarms package. Whether you are a beginner, intermediate, or expert developer, this tutorial will provide you with a clear understanding of how to effectively use the SequentialWorkflow class in your projects. AI engineering is a dynamic and evolving field that involves the development and deployment of intelligent systems and applications. In this ever-changing landscape, AI engineers often face the challenge of orchestrating complex sequences of tasks, managing data flows, and ensuring the smooth execution of AI workflows. This is where the Workflow Class, such as the SequentialWorkflow class we discussed earlier, plays a pivotal role in enabling AI engineers to achieve their goals efficiently and effectively. -The Versatile World of AI Workflows +## The Versatile World of AI Workflows AI workflows encompass a wide range of tasks and processes, from data preprocessing and model training to natural language understanding and decision-making. These workflows are the backbone of AI systems, guiding them through intricate sequences of actions to deliver meaningful results. Here are some of the diverse use cases where the Workflow Class can empower AI engineers: -1. Natural Language Processing (NLP) Pipelines +### 1. Natural Language Processing (NLP) Pipelines AI engineers often build NLP pipelines that involve multiple stages such as text preprocessing, tokenization, feature extraction, model inference, and post-processing. The Workflow Class enables the orderly execution of these stages, ensuring that textual data flows seamlessly through each step, resulting in accurate and coherent NLP outcomes. -2. Data Ingestion and Transformation +### 2. Data Ingestion and Transformation AI projects frequently require the ingestion of diverse data sources, including structured databases, unstructured text, and multimedia content. The Workflow Class can be used to design data ingestion workflows that extract, transform, and load (ETL) data efficiently, making it ready for downstream AI tasks like training and analysis. -3. Autonomous Agents and Robotics +### 3. Autonomous Agents and Robotics In autonomous robotics and intelligent agent systems, workflows are essential for decision-making, sensor fusion, motion planning, and control. AI engineers can use the Workflow Class to create structured sequences of actions that guide robots and agents through dynamic environments, enabling them to make informed decisions and accomplish tasks autonomously. -4. Machine Learning Model Training +### 4. Machine Learning Model Training Training machine learning models involves a series of steps, including data preprocessing, feature engineering, model selection, hyperparameter tuning, and evaluation. The Workflow Class simplifies the orchestration of these steps, allowing AI engineers to experiment with different configurations and track the progress of model training. -5. Content Generation and Summarization +### 5. Content Generation and Summarization AI-driven content generation tasks, such as generating articles, reports, or summaries, often require multiple steps, including content creation and post-processing. The Workflow Class can be used to create content generation workflows, ensuring that the generated content meets quality and coherence criteria. -6. Adaptive Decision-Making +### 6. Adaptive Decision-Making In AI systems that make real-time decisions based on changing data and environments, workflows facilitate adaptive decision-making. Engineers can use the Workflow Class to design decision-making pipelines that take into account the latest information and make informed choices. -Enabling Efficiency and Maintainability +## Enabling Efficiency and Maintainability The Workflow Class provides AI engineers with a structured and maintainable approach to building, executing, and managing complex AI workflows. It offers the following advantages: -Modularity: Workflows can be modularly designed, allowing engineers to focus on individual task implementations and ensuring code reusability. +- Modularity: Workflows can be modularly designed, allowing engineers to focus on individual task implementations and ensuring code reusability. -Debugging and Testing: The Workflow Class simplifies debugging and testing by providing a clear sequence of tasks and well-defined inputs and outputs for each task. +- Debugging and Testing: The Workflow Class simplifies debugging and testing by providing a clear sequence of tasks and well-defined inputs and outputs for each task. -Scalability: As AI projects grow in complexity, the Workflow Class can help manage and scale workflows by adding or modifying tasks as needed. +- Scalability: As AI projects grow in complexity, the Workflow Class can help manage and scale workflows by adding or modifying tasks as needed. -Error Handling: The class supports error handling strategies, enabling engineers to define how to handle unexpected failures gracefully. +- Error Handling: The class supports error handling strategies, enabling engineers to define how to handle unexpected failures gracefully. -Maintainability: With structured workflows, AI engineers can easily maintain and update AI systems as requirements evolve or new data sources become available. +- Maintainability: With structured workflows, AI engineers can easily maintain and update AI systems as requirements evolve or new data sources become available. The Workflow Class, such as the SequentialWorkflow class, is an indispensable tool in the toolkit of AI engineers. It empowers engineers to design, execute, and manage AI workflows across a diverse range of use cases. By providing structure, modularity, and maintainability to AI projects, the Workflow Class contributes significantly to the efficiency and success of AI engineering endeavors. As the field of AI continues to advance, harnessing the power of workflow orchestration will remain a key ingredient in building intelligent and adaptable systems, now let’s get started with SequentialWorkflow. @@ -142,71 +142,81 @@ In this example, we update the max_loops argument of the task with the descripti # Conclusion: Mastering Workflow Orchestration in AI Engineering In the ever-evolving landscape of artificial intelligence (AI), where the pace of innovation and complexity of tasks are ever-increasing, harnessing the power of workflow orchestration is paramount. In this comprehensive walkthrough guide, we’ve embarked on a journey through the world of workflow orchestration, focusing on the Workflow Class, with a specific emphasis on the SequentialWorkflow class. As we conclude this exploration, we’ve delved deep into the intricacies of orchestrating AI workflows, and it’s time to reflect on the valuable insights gained and the immense potential that this knowledge unlocks for AI engineers. -The Art of Workflow Orchestration +## The Art of Workflow Orchestration At its core, workflow orchestration is the art of designing, managing, and executing sequences of tasks or processes in a structured and efficient manner. In the realm of AI engineering, where tasks can range from data preprocessing and model training to decision-making and autonomous actions, mastering workflow orchestration is a game-changer. It empowers AI engineers to streamline their work, ensure reliable execution, and deliver impactful results. The Workflow Class, and particularly the SequentialWorkflow class we’ve explored, acts as a guiding light in this intricate journey. It provides AI engineers with a toolbox of tools and techniques to conquer the challenges of orchestrating AI workflows effectively. Through a disciplined approach and adherence to best practices, AI engineers can achieve the following: -1. Structured Workflow Design +### 1. Structured Workflow Design A well-structured workflow is the cornerstone of any successful AI project. The Workflow Class encourages AI engineers to break down complex tasks into manageable units. Each task becomes a building block that contributes to the overarching goal. Whether it’s preprocessing data, training a machine learning model, or generating content, structured workflow design ensures clarity, modularity, and maintainability. -2. Efficient Task Sequencing +### 2. Efficient Task Sequencing In AI, the order of tasks often matters. One task’s output can be another task’s input, and ensuring the correct sequence of execution is crucial. The SequentialWorkflow class enforces this sequential execution, eliminating the risk of running tasks out of order. It ensures that the workflow progresses systematically, following the predefined sequence of tasks. -3. Error Resilience and Recovery +### 3. Error Resilience and Recovery AI systems must be resilient in the face of unexpected errors and failures. The Workflow Class equips AI engineers with error handling strategies, such as retries and fallbacks. These strategies provide the ability to gracefully handle issues, recover from failures, and continue the workflow’s execution without disruption. -4. Code Modularity and Reusability +### 4. Code Modularity and Reusability Building AI workflows often involves implementing various tasks, each with its own logic. The Workflow Class encourages code modularity, allowing AI engineers to encapsulate tasks as separate units. This modularity promotes code reusability, making it easier to adapt and expand workflows as AI projects evolve. -5. Efficient Debugging and Testing +### 5. Efficient Debugging and Testing Debugging and testing AI workflows can be challenging without clear structure and boundaries. The Workflow Class provides a clear sequence of tasks with well-defined inputs and outputs. This structure simplifies the debugging process, as AI engineers can isolate and test individual tasks, ensuring that each component functions as intended. -6. Scalability and Adaptability +### 6. Scalability and Adaptability As AI projects grow in complexity, the Workflow Class scales effortlessly. AI engineers can add or modify tasks as needed, accommodating new data sources, algorithms, or requirements. This scalability ensures that workflows remain adaptable to changing demands and evolving AI landscapes. -7. Maintainability and Future-Proofing +### 7. Maintainability and Future-Proofing Maintaining AI systems over time is a crucial aspect of engineering. The Workflow Class fosters maintainability by providing a clear roadmap of tasks and their interactions. AI engineers can revisit, update, and extend workflows with confidence, ensuring that AI systems remain effective and relevant in the long run. -Empowering AI Engineers +## Empowering AI Engineers The knowledge and skills gained from this walkthrough guide go beyond technical proficiency. They empower AI engineers to be architects of intelligent systems, capable of orchestrating AI workflows that solve real-world problems. The Workflow Class is a versatile instrument in their hands, enabling them to tackle diverse use cases and engineering challenges. -Diverse Use Cases for Workflow Class +## Diverse Use Cases for Workflow Class Throughout this guide, we explored a myriad of use cases where the Workflow Class shines: Natural Language Processing (NLP) Pipelines: In NLP, workflows involve multiple stages, and the Workflow Class ensures orderly execution, resulting in coherent NLP outcomes. + Data Ingestion and Transformation: Data is the lifeblood of AI, and structured data workflows ensure efficient data preparation for downstream tasks. + Autonomous Agents and Robotics: For robots and intelligent agents, workflows enable autonomous decision-making and task execution. + Machine Learning Model Training: Model training workflows encompass numerous steps, and structured orchestration simplifies the process. + Content Generation and Summarization: Workflows for content generation ensure that generated content meets quality and coherence criteria. + Adaptive Decision-Making: In dynamic environments, workflows facilitate adaptive decision-making based on real-time data. -Efficiency and Maintainability + +## Efficiency and Maintainability AI engineers not only have the tools to tackle these use cases but also the means to do so efficiently. The Workflow Class fosters efficiency and maintainability, making AI engineering endeavors more manageable: -Modularity: Encapsulate tasks as separate units, promoting code reusability and maintainability. -Debugging and Testing: Streamline debugging and testing through clear task boundaries and well-defined inputs and outputs. -Scalability: As AI projects grow, workflows scale with ease, accommodating new components and requirements. +- Modularity: Encapsulate tasks as separate units, promoting code reusability and maintainability. + +- Debugging and Testing: Streamline debugging and testing through clear task boundaries and well-defined inputs and outputs. + +- Scalability: As AI projects grow, workflows scale with ease, accommodating new components and requirements. Error Handling: Gracefully handle errors and failures, ensuring that AI systems continue to operate smoothly. -Maintainability: AI systems remain adaptable and maintainable, even as the AI landscape evolves and requirements change. -The Future of AI Engineering + +- Maintainability: AI systems remain adaptable and maintainable, even as the AI landscape evolves and requirements change. + +## The Future of AI Engineering As AI engineering continues to advance, workflow orchestration will play an increasingly pivotal role. The Workflow Class is not a static tool; it is a dynamic enabler of innovation. In the future, we can expect further enhancements and features to meet the evolving demands of AI engineering: -1. Asynchronous Support +### 1. Asynchronous Support Support for asynchronous task execution will improve the efficiency of workflows, especially when tasks involve waiting for external events or resources. -2. Context Managers +### 2. Context Managers Introducing context manager support for tasks can simplify resource management, such as opening and closing files or database connections. -3. Workflow History +### 3. Workflow History Maintaining a detailed history of workflow execution, including timestamps, task durations, and input/output data, will facilitate debugging and performance analysis. -4. Parallel Processing +### 4. Parallel Processing Enhancing the module to support parallel processing with a pool of workers can significantly speed up the execution of tasks, especially for computationally intensive workflows. -5. Error Handling Strategies +### 5. Error Handling Strategies Providing built-in error handling strategies, such as retries, fallbacks, and circuit breakers, will further enhance the resilience of workflows. -Closing Thoughts +## Closing Thoughts In conclusion, the journey through workflow orchestration in AI engineering has been both enlightening and empowering. The Workflow Class, and particularly the SequentialWorkflow class, has proven to be an invaluable ally in the AI engineer’s toolkit. It offers structure, modularity, and efficiency, ensuring that AI projects progress smoothly from inception to deployment. As AI continues to permeate every aspect of our lives, the skills acquired in this guide will remain highly relevant and sought after. AI engineers armed with workflow orchestration expertise will continue to push the boundaries of what is possible, solving complex problems, and driving innovation. diff --git a/mkdocs.yml b/mkdocs.yml index abd2bd42..f7fd7f90 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,20 +61,6 @@ nav: - Home: - Overview: "index.md" - Contributing: "contributing.md" - - FAQ: "faq.md" - - Purpose: "purpose.md" - - Roadmap: "roadmap.md" - - Weaknesses: "failures.md" - - Design: "design.md" - - Flywheel: "flywheel.md" - - Bounties: "bounties.md" - - Metric: "metric.md" - - Distribution: "distribution" - - Research: "research.md" - - Demos: "demos.md" - - Architecture: "architecture.md" - - Checklist: "checklist.md" - - Hiring: "hiring.md" - Swarms: - Overview: "swarms/index.md" - swarms.swarms: @@ -122,11 +108,27 @@ nav: - Overview: "examples/index.md" - Structs: - Flow: "examples/flow.md" + - SequentialWorkflow: "examples/reliable_autonomous_agents.md" - Agents: - OmniAgent: "examples/omni_agent.md" - Worker: - Basic: "examples/worker.md" - StackedWorker: "examples/stacked_worker.md" +- Corporate: + - FAQ: "faq.md" + - Purpose: "purpose.md" + - Roadmap: "roadmap.md" + - Weaknesses: "failures.md" + - Design: "design.md" + - Flywheel: "flywheel.md" + - Bounties: "bounties.md" + - Metric: "metric.md" + - Distribution: "distribution" + - Research: "research.md" + - Demos: "demos.md" + - Architecture: "architecture.md" + - Checklist: "checklist.md" + - Hiring: "hiring.md" - Applications: - CustomerSupport: - Overview: "applications/customer_support.md" From c520cda250ab96f27fc26e602418c339f0fee313 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 12:37:35 -0500 Subject: [PATCH 18/32] multi modal auto agent + removed workflow.py --- mkdocs.yml | 12 ++++++------ multi_modal_auto_agent.py | 30 ++++++++++++++++++++++++++++++ swarms/structs/base.py | 5 +++++ swarms/structs/flow.py | 24 +++++++++++++----------- swarms/swarms/base.py | 2 -- workflow.py | 11 ----------- 6 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 multi_modal_auto_agent.py create mode 100644 swarms/structs/base.py delete mode 100644 workflow.py diff --git a/mkdocs.yml b/mkdocs.yml index f7fd7f90..e3c93d94 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -114,6 +114,11 @@ nav: - Worker: - Basic: "examples/worker.md" - StackedWorker: "examples/stacked_worker.md" +- Applications: + - CustomerSupport: + - Overview: "applications/customer_support.md" + - Marketing: + - Overview: "applications/marketing_agencies.md" - Corporate: - FAQ: "faq.md" - Purpose: "purpose.md" @@ -128,9 +133,4 @@ nav: - Demos: "demos.md" - Architecture: "architecture.md" - Checklist: "checklist.md" - - Hiring: "hiring.md" -- Applications: - - CustomerSupport: - - Overview: "applications/customer_support.md" - - Marketing: - - Overview: "applications/marketing_agencies.md" + - Hiring: "hiring.md" \ No newline at end of file diff --git a/multi_modal_auto_agent.py b/multi_modal_auto_agent.py new file mode 100644 index 00000000..b462795f --- /dev/null +++ b/multi_modal_auto_agent.py @@ -0,0 +1,30 @@ +from swarms.structs import Flow +from swarms.models import Idefics + +# Multi Modality Auto Agent +llm = Idefics(max_length=2000) + +task = "User: What is in this image? https://upload.wikimedia.org/wikipedia/commons/8/86/Id%C3%A9fix.JPG" + +## Initialize the workflow +flow = Flow( + llm=llm, + max_loops=2, + dashboard=True, + # stopping_condition=None, # You can define a stopping condition as needed. + # loop_interval=1, + # retry_attempts=3, + # retry_interval=1, + # interactive=False, # Set to 'True' for interactive mode. + # dynamic_temperature=False, # Set to 'True' for dynamic temperature handling. +) + +# out = flow.load_state("flow_state.json") +# temp = flow.dynamic_temperature() +# filter = flow.add_response_filter("Trump") +out = flow.run(task) +# out = flow.validate_response(out) +# out = flow.analyze_feedback(out) +# out = flow.print_history_and_memory() +# # out = flow.save_state("flow_state.json") +# print(out) diff --git a/swarms/structs/base.py b/swarms/structs/base.py new file mode 100644 index 00000000..f33a204e --- /dev/null +++ b/swarms/structs/base.py @@ -0,0 +1,5 @@ +""" +Base Structure for all Swarm Structures + + +""" \ No newline at end of file diff --git a/swarms/structs/flow.py b/swarms/structs/flow.py index 0f129314..117172ea 100644 --- a/swarms/structs/flow.py +++ b/swarms/structs/flow.py @@ -111,10 +111,10 @@ class Flow: interactive: bool = False, dashboard: bool = False, name: str = "Flow agent", - system_message: str = FLOW_SYSTEM_PROMPT, + system_prompt: str = FLOW_SYSTEM_PROMPT, # tools: List[BaseTool] = None, dynamic_temperature: bool = False, - saved_state: Optional[str] = None, + saved_state_path: Optional[str] = "flow_state.json", autosave: bool = False, **kwargs: Any, ): @@ -133,9 +133,9 @@ class Flow: self.dashboard = dashboard self.dynamic_temperature = dynamic_temperature # self.tools = tools - self.system_message = system_message + self.system_prompt = system_prompt self.name = name - self.saved_state = saved_state + self.saved_state_path = saved_state_path self.autosave = autosave self.response_filters = [] @@ -206,7 +206,7 @@ class Flow: Flow Configuration: Name: {self.name} - System Prompt: {self.system_message} + System Prompt: {self.system_prompt} Task: {task} Max Loops: {self.max_loops} Stopping Condition: {self.stopping_condition} @@ -317,8 +317,10 @@ class Flow: time.sleep(self.loop_interval) self.memory.append(history) - # if self.autosave: - # self.save_state("flow_state.json") + if self.autosave: + save_path = self.saved_state_path or "flow_state.json" + print(colored(f"Autosaving flow state to {save_path}", "green")) + self.save_state(save_path) return response # , history @@ -422,7 +424,7 @@ class Flow: Returns: str: The agent history prompt """ - system_prompt = system_prompt or self.system_message + system_prompt = system_prompt or self.system_prompt agent_history_prompt = f""" SYSTEM_PROMPT: {system_prompt} @@ -736,7 +738,7 @@ class Flow: """ prompt = f""" - SYSTEM_PROMPT: {self.system_message} + SYSTEM_PROMPT: {self.system_prompt} History: {history} @@ -745,6 +747,6 @@ class Flow: response = self.llm(prompt, **kwargs) return {"role": self.name, "content": response} - def update_system_message(self, system_message: str): + def update_system_prompt(self, system_prompt: str): """Upddate the system message""" - self.system_message = system_message + self.system_prompt = system_prompt diff --git a/swarms/swarms/base.py b/swarms/swarms/base.py index 21f30ae3..e99c9b38 100644 --- a/swarms/swarms/base.py +++ b/swarms/swarms/base.py @@ -78,8 +78,6 @@ class AbstractSwarm(ABC): Scale down the number of workers - - """ # TODO: Pass in abstract LLM class that can utilize Hf or Anthropic models, Move away from OPENAI diff --git a/workflow.py b/workflow.py deleted file mode 100644 index bc757108..00000000 --- a/workflow.py +++ /dev/null @@ -1,11 +0,0 @@ -from swarms.models import OpenAIChat -from swarms.structs import Workflow - - -llm = OpenAIChat(openai_api_key="") - -workflow = Workflow(llm) - -workflow.add("What's the weather in miami") - -workflow.run() From 881ec11f07e786c50ad631ac5a527d0cc3c61742 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 12:56:17 -0500 Subject: [PATCH 19/32] docs clean up -> corporate folder --- docs/{ => corporate}/architecture.md | 0 docs/{ => corporate}/bounties.md | 0 docs/{ => corporate}/checklist.md | 0 docs/{ => corporate}/cost_analysis.md | 0 docs/{ => corporate}/demos.md | 0 docs/{ => corporate}/design.md | 0 docs/{ => corporate}/distribution.md | 0 docs/{ => corporate}/failures.md | 0 docs/{ => corporate}/faq.md | 0 docs/{ => corporate}/flywheel.md | 0 docs/{ => corporate}/hiring.md | 0 docs/{ => corporate}/metric.md | 0 docs/{ => corporate}/purpose.md | 0 docs/{ => corporate}/research.md | 0 docs/{ => corporate}/roadmap.md | 0 docs/examples/ideas.md | 63 +++++++++++++++++++++++++++ docs/swarms/models/fuyu.md | 7 --- docs/swarms/swarms/groupchat.md | 0 mkdocs.yml | 30 +++++++------ swarms/utils/code_interpreter.py | 11 ++++- 20 files changed, 89 insertions(+), 22 deletions(-) rename docs/{ => corporate}/architecture.md (100%) rename docs/{ => corporate}/bounties.md (100%) rename docs/{ => corporate}/checklist.md (100%) rename docs/{ => corporate}/cost_analysis.md (100%) rename docs/{ => corporate}/demos.md (100%) rename docs/{ => corporate}/design.md (100%) rename docs/{ => corporate}/distribution.md (100%) rename docs/{ => corporate}/failures.md (100%) rename docs/{ => corporate}/faq.md (100%) rename docs/{ => corporate}/flywheel.md (100%) rename docs/{ => corporate}/hiring.md (100%) rename docs/{ => corporate}/metric.md (100%) rename docs/{ => corporate}/purpose.md (100%) rename docs/{ => corporate}/research.md (100%) rename docs/{ => corporate}/roadmap.md (100%) create mode 100644 docs/examples/ideas.md create mode 100644 docs/swarms/swarms/groupchat.md diff --git a/docs/architecture.md b/docs/corporate/architecture.md similarity index 100% rename from docs/architecture.md rename to docs/corporate/architecture.md diff --git a/docs/bounties.md b/docs/corporate/bounties.md similarity index 100% rename from docs/bounties.md rename to docs/corporate/bounties.md diff --git a/docs/checklist.md b/docs/corporate/checklist.md similarity index 100% rename from docs/checklist.md rename to docs/corporate/checklist.md diff --git a/docs/cost_analysis.md b/docs/corporate/cost_analysis.md similarity index 100% rename from docs/cost_analysis.md rename to docs/corporate/cost_analysis.md diff --git a/docs/demos.md b/docs/corporate/demos.md similarity index 100% rename from docs/demos.md rename to docs/corporate/demos.md diff --git a/docs/design.md b/docs/corporate/design.md similarity index 100% rename from docs/design.md rename to docs/corporate/design.md diff --git a/docs/distribution.md b/docs/corporate/distribution.md similarity index 100% rename from docs/distribution.md rename to docs/corporate/distribution.md diff --git a/docs/failures.md b/docs/corporate/failures.md similarity index 100% rename from docs/failures.md rename to docs/corporate/failures.md diff --git a/docs/faq.md b/docs/corporate/faq.md similarity index 100% rename from docs/faq.md rename to docs/corporate/faq.md diff --git a/docs/flywheel.md b/docs/corporate/flywheel.md similarity index 100% rename from docs/flywheel.md rename to docs/corporate/flywheel.md diff --git a/docs/hiring.md b/docs/corporate/hiring.md similarity index 100% rename from docs/hiring.md rename to docs/corporate/hiring.md diff --git a/docs/metric.md b/docs/corporate/metric.md similarity index 100% rename from docs/metric.md rename to docs/corporate/metric.md diff --git a/docs/purpose.md b/docs/corporate/purpose.md similarity index 100% rename from docs/purpose.md rename to docs/corporate/purpose.md diff --git a/docs/research.md b/docs/corporate/research.md similarity index 100% rename from docs/research.md rename to docs/corporate/research.md diff --git a/docs/roadmap.md b/docs/corporate/roadmap.md similarity index 100% rename from docs/roadmap.md rename to docs/corporate/roadmap.md diff --git a/docs/examples/ideas.md b/docs/examples/ideas.md new file mode 100644 index 00000000..a0a9c9b7 --- /dev/null +++ b/docs/examples/ideas.md @@ -0,0 +1,63 @@ +# 2O+ Autonomous Agent Blogs + +1. **The Ultimate Guide to Deploying Production-Ready Autonomous Agents with Swarms** + - A comprehensive start-to-finish guide on implementing Swarms in a production environment. + +2. **5 Steps to Elevate Your AI with Swarms Multi-Modal Autonomous Agents** + - A walkthrough highlighting the simplicity of Swarms’ setup and deployment for various AI applications. + +3. **Integrating Swarms Into Your Enterprise Workflow: A Step-By-Step Tutorial** + - A practical guide focusing on integrating Swarms into existing enterprise systems. + +4. **Swarms’ Flow: Streamlining AI Deployment in Your Business** + - Exploring the benefits and technicalities of using the Flow feature to simplify complex AI workflows. + +5. **From Zero to Hero: Building Your First Enterprise-Grade AI Agent with Swarms** + - A beginner-friendly walkthrough for building and deploying an AI agent using Swarms. + +6. **Scaling AI with Swarms: Managing Multi-Agent Systems Efficiently** + - Strategies and best practices for scaling multi-agent systems in enterprise settings. + +7. **Creating Resilient AI Systems with Swarms' Autonomous Agents** + - Discussing the robustness of Swarms agents and how they maintain performance under stress. + +8. **Unlocking New Capabilities: Advanced Features of Swarms for AI Engineers** + - Diving into the more sophisticated features of Swarms and how they can be leveraged in complex projects. + +9. **Swarms’ Quick Wins: Implementing AI Agents in Less Than 5 Lines of Code** + - A focused guide on rapidly deploying functional AI agents with minimal coding. + +10. **Benchmarking Your AI: Performance Metrics with Swarms** + - How to use Swarms to measure and optimize the performance of AI agents. + +11. **Swarms Case Studies: Real-World Success Stories from AI Engineers** + - Sharing stories and testimonials of how various organizations successfully implemented Swarms. + +12. **Effortless Multi-Modal Model Deployment: A Swarms Walkthrough** + - Explaining how to use Swarms to deploy multi-modal models with ease. + +13. **Future-Proof Your AI: Adapting to New Tech with Swarms** + - How Swarms' flexible architecture allows for easy updates and adaptation to new AI technologies. + +14. **Enterprise AI Security: Ensuring Your Swarms Agents are Hack-Proof** + - Best practices for securing autonomous agents in enterprise applications. + +15. **Migrating to Swarms: Transitioning From Legacy Systems** + - A guide for AI engineers on migrating existing AI systems to Swarms without downtime. + +16. **Multi-Agent Collaboration: How Swarms Facilitates Teamwork Among AI** + - An insight into how Swarms allows for multiple AI agents to work together seamlessly. + +17. **The Engineer's Toolkit: Swarms' Features Every AI Developer Must Know** + - Highlighting the most useful tools and features of Swarms from an AI developer’s perspective. + +18. **Swarms for Different Industries: Customizing AI Agents for Niche Markets** + - Exploring how Swarms can be tailored to fit the needs of various industries such as healthcare, finance, and retail. + +19. **Building Intelligent Workflows with Swarms’ Flow** + - A tutorial on using the Flow feature to create intelligent, responsive AI-driven workflows. + +20. **Troubleshooting Common Issues When Deploying Swarms Autonomous Agents** + - A problem-solving guide for AI engineers on overcoming common challenges when implementing Swarms agents. + +Each blog or walkthrough can be structured to not only showcase the functionality and benefits of the Swarms framework but also to establish the brand as a thought leader in the space of enterprise AI solutions. \ No newline at end of file diff --git a/docs/swarms/models/fuyu.md b/docs/swarms/models/fuyu.md index e342e51e..021469e8 100644 --- a/docs/swarms/models/fuyu.md +++ b/docs/swarms/models/fuyu.md @@ -42,13 +42,6 @@ from swarms.models import Fuyu fuyu = Fuyu() ``` -### Example 1 - Initialization - -```python -from swarms.models import Fuyu - -fuyu = Fuyu() -``` 2. Generate Text with Fuyu: diff --git a/docs/swarms/swarms/groupchat.md b/docs/swarms/swarms/groupchat.md new file mode 100644 index 00000000..e69de29b diff --git a/mkdocs.yml b/mkdocs.yml index e3c93d94..aff83631 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,7 @@ nav: - AbstractSwarm: "swarms/swarms/abstractswarm.md" - AutoScaler: "swarms/swarms/autoscaler.md" - GodMode: "swarms/swarms/godmode.md" + - Groupchat: "swarms/swarms/groupchat.md" - swarms.workers: - AbstractWorker: "swarms/workers/base.md" - Overview: "swarms/workers/index.md" @@ -114,23 +115,24 @@ nav: - Worker: - Basic: "examples/worker.md" - StackedWorker: "examples/stacked_worker.md" + - 2O+ Autonomous Agent Blogs: "examples/ideas.md" - Applications: - CustomerSupport: - Overview: "applications/customer_support.md" - Marketing: - Overview: "applications/marketing_agencies.md" - Corporate: - - FAQ: "faq.md" - - Purpose: "purpose.md" - - Roadmap: "roadmap.md" - - Weaknesses: "failures.md" - - Design: "design.md" - - Flywheel: "flywheel.md" - - Bounties: "bounties.md" - - Metric: "metric.md" - - Distribution: "distribution" - - Research: "research.md" - - Demos: "demos.md" - - Architecture: "architecture.md" - - Checklist: "checklist.md" - - Hiring: "hiring.md" \ No newline at end of file + - FAQ: "corporate/faq.md" + - Purpose: "corporate/purpose.md" + - Roadmap: "corporate/roadmap.md" + - Weaknesses: "corporate/failures.md" + - Design: "corporate/design.md" + - Flywheel: "corporate/flywheel.md" + - Bounties: "corporate/bounties.md" + - Metric: "corporate/metric.md" + - Distribution: "corporate/distribution" + - Research: "corporate/research.md" + - Demos: "corporate/demos.md" + - Architecture: "corporate/architecture.md" + - Checklist: "corporate/checklist.md" + - Hiring: "corporate/hiring.md" diff --git a/swarms/utils/code_interpreter.py b/swarms/utils/code_interpreter.py index cf557385..af6eb327 100644 --- a/swarms/utils/code_interpreter.py +++ b/swarms/utils/code_interpreter.py @@ -24,7 +24,16 @@ class SubprocessCodeInterpreter(BaseCodeInterpreter): """ SubprocessCodeinterpreter is a base class for code interpreters that run code in a subprocess. - + + Attributes: + start_cmd (str): The command to start the subprocess. Should be a string that can be split by spaces. + process (subprocess.Popen): The subprocess that is running the code. + debug_mode (bool): Whether to print debug statements. + output_queue (queue.Queue): A queue that is filled with output from the subprocess. + done (threading.Event): An event that is set when the subprocess is done running code. + + Example: + >>> from swarms.utils.code_interpreter import SubprocessCodeInterpreter """ From a97f759ae50a93b5747df65262fa9f388ec6b0a2 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 12:57:39 -0500 Subject: [PATCH 20/32] m --- mkdocs.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index aff83631..338b0cda 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -107,14 +107,10 @@ nav: - PdfChunker: "swarms/chunkers/pdf_chunker.md" - Walkthroughs: - Overview: "examples/index.md" - - Structs: + - Agents: - Flow: "examples/flow.md" - SequentialWorkflow: "examples/reliable_autonomous_agents.md" - - Agents: - OmniAgent: "examples/omni_agent.md" - - Worker: - - Basic: "examples/worker.md" - - StackedWorker: "examples/stacked_worker.md" - 2O+ Autonomous Agent Blogs: "examples/ideas.md" - Applications: - CustomerSupport: From fe48ec1393fdad65221dd9238eda95db503da899 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 12:57:47 -0500 Subject: [PATCH 21/32] guides --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 338b0cda..58430091 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -105,7 +105,7 @@ nav: - swarms.chunkers: - BaseChunker: "swarms/chunkers/basechunker.md" - PdfChunker: "swarms/chunkers/pdf_chunker.md" -- Walkthroughs: +- Guides: - Overview: "examples/index.md" - Agents: - Flow: "examples/flow.md" From fd8919dde5f1b38b823fd4862046f051154abb63 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 16:23:49 -0500 Subject: [PATCH 22/32] GPT4Vision + Dalle3 -> modules + tests + documentation --- dalle3.py | 6 + docs/swarms/models/gpt4v.md | 251 ++++++++++++++++++ gpt4vision_example.py | 7 + sequential_workflow_example.py | 4 +- swarms/models/dalle3.py | 175 +++++++++++++ swarms/models/gpt4v.py | 288 +++++++++++++++++++++ swarms/structs/base.py | 2 +- swarms/utils/code_interpreter.py | 2 +- tests/models/dalle3.py | 374 +++++++++++++++++++++++++++ tests/models/gpt4v.py | 321 +++++++++++++++++++++++ tests/structs/sequential_workflow.py | 35 ++- 11 files changed, 1456 insertions(+), 9 deletions(-) create mode 100644 dalle3.py create mode 100644 docs/swarms/models/gpt4v.md create mode 100644 gpt4vision_example.py create mode 100644 swarms/models/dalle3.py create mode 100644 swarms/models/gpt4v.py create mode 100644 tests/models/dalle3.py create mode 100644 tests/models/gpt4v.py diff --git a/dalle3.py b/dalle3.py new file mode 100644 index 00000000..ac9ba760 --- /dev/null +++ b/dalle3.py @@ -0,0 +1,6 @@ +from swarms.models.dalle3 import Dalle3 + +model = Dalle3() + +task = "A painting of a dog" +img = model(task) diff --git a/docs/swarms/models/gpt4v.md b/docs/swarms/models/gpt4v.md new file mode 100644 index 00000000..2af4348b --- /dev/null +++ b/docs/swarms/models/gpt4v.md @@ -0,0 +1,251 @@ +# GPT4Vision Documentation + +## Table of Contents +- [Overview](#overview) +- [Installation](#installation) +- [Initialization](#initialization) +- [Methods](#methods) + - [process_img](#process_img) + - [__call__](#__call__) + - [run](#run) + - [arun](#arun) +- [Configuration Options](#configuration-options) +- [Usage Examples](#usage-examples) +- [Additional Tips](#additional-tips) +- [References and Resources](#references-and-resources) + +--- + +## Overview + +The GPT4Vision Model API is designed to provide an easy-to-use interface for interacting with the OpenAI GPT-4 Vision model. This model can generate textual descriptions for images and answer questions related to visual content. Whether you want to describe images or perform other vision-related tasks, GPT4Vision makes it simple and efficient. + +The library offers a straightforward way to send images and tasks to the GPT-4 Vision model and retrieve the generated responses. It handles API communication, authentication, and retries, making it a powerful tool for developers working with computer vision and natural language processing tasks. + +## Installation + +To use the GPT4Vision Model API, you need to install the required dependencies and configure your environment. Follow these steps to get started: + +1. Install the required Python package: + + ```bash + pip3 install --upgrade swarms + ``` + +2. Make sure you have an OpenAI API key. You can obtain one by signing up on the [OpenAI platform](https://beta.openai.com/signup/). + +3. Set your OpenAI API key as an environment variable. You can do this in your code or your environment configuration. Alternatively, you can provide the API key directly when initializing the `GPT4Vision` class. + +## Initialization + +To start using the GPT4Vision Model API, you need to create an instance of the `GPT4Vision` class. You can customize its behavior by providing various configuration options, but it also comes with sensible defaults. + +Here's how you can initialize the `GPT4Vision` class: + +```python +from swarms.models.gpt4v import GPT4Vision + +gpt4vision = GPT4Vision( + api_key="Your Key" +) +``` + +The above code initializes the `GPT4Vision` class with default settings. You can adjust these settings as needed. + +## Methods + +### `process_img` + +The `process_img` method is used to preprocess an image before sending it to the GPT-4 Vision model. It takes the image path as input and returns the processed image in a format suitable for API requests. + +```python +processed_img = gpt4vision.process_img(img_path) +``` + +- `img_path` (str): The file path or URL of the image to be processed. + +### `__call__` + +The `__call__` method is the main method for interacting with the GPT-4 Vision model. It sends the image and tasks to the model and returns the generated response. + +```python +response = gpt4vision(img, tasks) +``` + +- `img` (Union[str, List[str]]): Either a single image URL or a list of image URLs to be used for the API request. +- `tasks` (List[str]): A list of tasks or questions related to the image(s). + +This method returns a `GPT4VisionResponse` object, which contains the generated answer. + +### `run` + +The `run` method is an alternative way to interact with the GPT-4 Vision model. It takes a single task and image URL as input and returns the generated response. + +```python +response = gpt4vision.run(task, img) +``` + +- `task` (str): The task or question related to the image. +- `img` (str): The image URL to be used for the API request. + +This method simplifies interactions when dealing with a single task and image. + +### `arun` + +The `arun` method is an asynchronous version of the `run` method. It allows for asynchronous processing of API requests, which can be useful in certain scenarios. + +```python +import asyncio + +async def main(): + response = await gpt4vision.arun(task, img) + print(response) + +loop = asyncio.get_event_loop() +loop.run_until_complete(main()) +``` + +- `task` (str): The task or question related to the image. +- `img` (str): The image URL to be used for the API request. + +## Configuration Options + +The `GPT4Vision` class provides several configuration options that allow you to customize its behavior: + +- `max_retries` (int): The maximum number of retries to make to the API. Default: 3 +- `backoff_factor` (float): The backoff factor to use for exponential backoff. Default: 2.0 +- `timeout_seconds` (int): The timeout in seconds for the API request. Default: 10 +- `api_key` (str): The API key to use for the API request. Default: None (set via environment variable) +- `quality` (str): The quality of the image to generate. Options: 'low' or 'high'. Default: 'low' +- `max_tokens` (int): The maximum number of tokens to use for the API request. Default: 200 + +## Usage Examples + +### Example 1: Generating Image Descriptions + +```python +gpt4vision = GPT4Vision() +img = "https://example.com/image.jpg" +tasks = ["Describe this image."] +response = gpt4vision(img, tasks) +print(response.answer) +``` + +In this example, we create an instance of `GPT4Vision`, provide an image URL, and ask the model to describe the image. The response contains the generated description. + +### Example 2: Custom Configuration + +```python +custom_config = { + "max_retries": 5, + "timeout_seconds": 20, + "quality": "high", + "max_tokens": 300, +} +gpt4vision = GPT4Vision(**custom_config) +img = "https://example.com/another_image.jpg" +tasks = ["What objects can you identify in this image?"] +response = gpt4vision(img, tasks) +print(response.answer) +``` + +In this example, we create an instance of `GPT4Vision` with custom configuration options. We set a higher timeout, request high-quality images, and allow more tokens in the response. + +### Example 3: Using the `run` Method + +```python +gpt4vision = GPT4Vision() +img = "https://example.com/image.jpg" +task = "Describe this image in detail." +response = gpt4vision.run(task, img) +print(response) +``` + +In this example, we use the `run` method to simplify the interaction by providing a single task and image URL. + +# Model Usage and Image Understanding + +The GPT-4 Vision model processes images in a unique way, allowing it to answer questions about both or each of the images independently. Here's an overview: + +| Purpose | Description | +| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| Image Understanding | The model is shown two copies of the same image and can answer questions about both or each of the images independently. | + +# Image Detail Control + +You have control over how the model processes the image and generates textual understanding by using the `detail` parameter, which has two options: `low` and `high`. + +| Detail | Description | +| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| low | Disables the "high-res" model. The model receives a low-res 512 x 512 version of the image and represents the image with a budget of 65 tokens. Ideal for use cases not requiring high detail. | +| high | Enables "high-res" mode. The model first sees the low-res image and then creates detailed crops of input images as 512px squares based on the input image size. Uses a total of 129 tokens. | + +# Managing Images + +To use the Chat Completions API effectively, you must manage the images you pass to the model. Here are some key considerations: + +| Management Aspect | Description | +| ------------------------- | ------------------------------------------------------------------------------------------------- | +| Image Reuse | To pass the same image multiple times, include the image with each API request. | +| Image Size Optimization | Improve latency by downsizing images to meet the expected size requirements. | +| Image Deletion | After processing, images are deleted from OpenAI servers and not retained. No data is used for training. | + +# Limitations + +While GPT-4 with Vision is powerful, it has some limitations: + +| Limitation | Description | +| -------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| Medical Images | Not suitable for interpreting specialized medical images like CT scans. | +| Non-English Text | May not perform optimally when handling non-Latin alphabets, such as Japanese or Korean. | +| Large Text in Images | Enlarge text within images for readability, but avoid cropping important details. | +| Rotated or Upside-Down Text/Images | May misinterpret rotated or upside-down text or images. | +| Complex Visual Elements | May struggle to understand complex graphs or text with varying colors or styles. | +| Spatial Reasoning | Struggles with tasks requiring precise spatial localization, such as identifying chess positions. | +| Accuracy | May generate incorrect descriptions or captions in certain scenarios. | +| Panoramic and Fisheye Images | Struggles with panoramic and fisheye images. | + +# Calculating Costs + +Image inputs are metered and charged in tokens. The token cost depends on the image size and detail option. + +| Example | Token Cost | +| --------------------------------------------- | ----------- | +| 1024 x 1024 square image in detail: high mode | 765 tokens | +| 2048 x 4096 image in detail: high mode | 1105 tokens | +| 4096 x 8192 image in detail: low mode | 85 tokens | + +# FAQ + +Here are some frequently asked questions about GPT-4 with Vision: + +| Question | Answer | +| -------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| Fine-Tuning Image Capabilities | No, fine-tuning the image capabilities of GPT-4 is not supported at this time. | +| Generating Images | GPT-4 is used for understanding images, not generating them. | +| Supported Image File Types | Supported image file types include PNG (.png), JPEG (.jpeg and .jpg), WEBP (.webp), and non-animated GIF (.gif). | +| Image Size Limitations | Image uploads are restricted to 20MB per image. | +| Image Deletion | Uploaded images are automatically deleted after processing by the model. | +| Learning More | For more details about GPT-4 with Vision, refer to the GPT-4 with Vision system card. | +| CAPTCHA Submission | CAPTCHAs are blocked for safety reasons. | +| Rate Limits | Image processing counts toward your tokens per minute (TPM) limit. Refer to the calculating costs section for details. | +| Image Metadata | The model does not receive image metadata. | +| Handling Unclear Images | If an image is unclear, the model will do its best to interpret it, but results may be less accurate. | + + + +## Additional Tips + +- Make sure to handle potential exceptions and errors when making API requests. The library includes retries and error handling, but it's essential to handle exceptions gracefully in your code. +- Experiment with different configuration options to optimize the trade-off between response quality and response time based on your specific requirements. + +## References and Resources + +- [OpenAI Platform](https://beta.openai.com/signup/): Sign up for an OpenAI API key. +- [OpenAI API Documentation](https://platform.openai.com/docs/api-reference/chat/create): Official API documentation for the GPT-4 Vision model. + +Now you have a comprehensive understanding of the GPT4Vision Model API, its configuration options, and how to use it for various computer vision and natural language processing tasks. Start experimenting and integrating it into your projects to leverage the power of GPT-4 Vision for image-related tasks. + +# Conclusion + +With GPT-4 Vision, you have a powerful tool for understanding and generating textual descriptions for images. By considering its capabilities, limitations, and cost calculations, you can effectively leverage this model for various image-related tasks. \ No newline at end of file diff --git a/gpt4vision_example.py b/gpt4vision_example.py new file mode 100644 index 00000000..7306fc56 --- /dev/null +++ b/gpt4vision_example.py @@ -0,0 +1,7 @@ +from swarms.models.gpt4v import GPT4Vision + +gpt4vision = GPT4Vision(api_key="") +task = "What is the following image about?" +img = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + +answer = gpt4vision.run(task, img) diff --git a/sequential_workflow_example.py b/sequential_workflow_example.py index b9ab8196..feb6c748 100644 --- a/sequential_workflow_example.py +++ b/sequential_workflow_example.py @@ -3,9 +3,7 @@ from swarms.structs import Flow from swarms.structs.sequential_workflow import SequentialWorkflow # Example usage -api_key = ( - "" # Your actual API key here -) +api_key = "" # Your actual API key here # Initialize the language flow llm = OpenAIChat( diff --git a/swarms/models/dalle3.py b/swarms/models/dalle3.py new file mode 100644 index 00000000..f22b11e0 --- /dev/null +++ b/swarms/models/dalle3.py @@ -0,0 +1,175 @@ +import openai +import logging +import os +from dataclasses import dataclass +from functools import lru_cache +from termcolor import colored +from openai import OpenAI +from dotenv import load_dotenv +from pydantic import BaseModel, validator +from PIL import Image +from io import BytesIO + + +load_dotenv() + +api_key = os.getenv("OPENAI_API_KEY") + +# Configure Logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +@dataclass +class Dalle3: + """ + Dalle3 model class + + Attributes: + ----------- + image_url: str + The image url generated by the Dalle3 API + + Methods: + -------- + __call__(self, task: str) -> Dalle3: + Makes a call to the Dalle3 API and returns the image url + + Example: + -------- + >>> dalle3 = Dalle3() + >>> task = "A painting of a dog" + >>> image_url = dalle3(task) + >>> print(image_url) + https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png + + """ + + model: str = "dall-e-3" + img: str = None + size: str = "1024x1024" + max_retries: int = 3 + quality: str = "standard" + n: int = 4 + client = OpenAI( + api_key=api_key, + max_retries=max_retries, + ) + + class Config: + """Config class for the Dalle3 model""" + + arbitrary_types_allowed = True + + @validator("max_retries", "time_seconds") + def must_be_positive(cls, value): + if value <= 0: + raise ValueError("Must be positive") + return value + + def read_img(self, img: str): + """Read the image using pil""" + img = Image.open(img) + return img + + def set_width_height(self, img: str, width: int, height: int): + """Set the width and height of the image""" + img = self.read_img(img) + img = img.resize((width, height)) + return img + + def convert_to_bytesio(self, img: str, format: str = "PNG"): + """Convert the image to an bytes io object""" + byte_stream = BytesIO() + img.save(byte_stream, format=format) + byte_array = byte_stream.getvalue() + return byte_array + + # @lru_cache(maxsize=32) + def __call__(self, task: str): + """ + Text to image conversion using the Dalle3 API + + Parameters: + ----------- + task: str + The task to be converted to an image + + Returns: + -------- + Dalle3: + An instance of the Dalle3 class with the image url generated by the Dalle3 API + + Example: + -------- + >>> dalle3 = Dalle3() + >>> task = "A painting of a dog" + >>> image_url = dalle3(task) + >>> print(image_url) + https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png + """ + try: + # Making a call to the the Dalle3 API + response = self.client.images.generate( + # model=self.model, + prompt=task, + # size=self.size, + # quality=self.quality, + n=self.n, + ) + # Extracting the image url from the response + img = response.data[0].url + return img + except openai.OpenAIError as error: + # Handling exceptions and printing the errors details + print( + colored( + f"Error running Dalle3: {error} try optimizing your api key and or try again", + "red", + ) + ) + raise error + + def create_variations(self, img: str): + """ + Create variations of an image using the Dalle3 API + + Parameters: + ----------- + img: str + The image to be used for the API request + + Returns: + -------- + img: str + The image url generated by the Dalle3 API + + Example: + -------- + >>> dalle3 = Dalle3() + >>> img = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + >>> img = dalle3.create_variations(img) + >>> print(img) + + + """ + try: + + response = self.client.images.create_variation( + img = open(img, "rb"), + n=self.n, + size=self.size + ) + img = response.data[0].url + + return img + except (Exception, openai.OpenAIError) as error: + print( + colored( + f"Error running Dalle3: {error} try optimizing your api key and or try again", + "red", + ) + ) + print(colored(f"Error running Dalle3: {error.http_status}", "red")) + print(colored(f"Error running Dalle3: {error.error}", "red")) + raise error \ No newline at end of file diff --git a/swarms/models/gpt4v.py b/swarms/models/gpt4v.py new file mode 100644 index 00000000..a7f8f1c1 --- /dev/null +++ b/swarms/models/gpt4v.py @@ -0,0 +1,288 @@ +import base64 +import logging +import os +import time +from dataclasses import dataclass +from typing import List, Optional, Union + +import requests +from dotenv import load_dotenv +from openai import OpenAI +from termcolor import colored + +# ENV +load_dotenv() + + +def logging_config(): + """Configures logging""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + logger = logging.getLogger(__name__) + + return logger + + +@dataclass +class GPT4VisionResponse: + """A response structure for GPT-4""" + + answer: str + + +@dataclass +class GPT4Vision: + """ + GPT4Vision model class + + Attributes: + ----------- + max_retries: int + The maximum number of retries to make to the API + backoff_factor: float + The backoff factor to use for exponential backoff + timeout_seconds: int + The timeout in seconds for the API request + api_key: str + The API key to use for the API request + quality: str + The quality of the image to generate + max_tokens: int + The maximum number of tokens to use for the API request + + Methods: + -------- + process_img(self, img_path: str) -> str: + Processes the image to be used for the API request + __call__(self, img: Union[str, List[str]], tasks: List[str]) -> GPT4VisionResponse: + Makes a call to the GPT-4 Vision API and returns the image url + + Example: + >>> gpt4vision = GPT4Vision() + >>> img = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + >>> tasks = ["A painting of a dog"] + >>> answer = gpt4vision(img, tasks) + >>> print(answer) + + + """ + + max_retries: int = 3 + model: str = "gpt-4-vision-preview" + backoff_factor: float = 2.0 + timeout_seconds: int = 10 + api_key: Optional[str] = None or os.getenv("OPENAI_API_KEY") + # 'Low' or 'High' for respesctively fast or high quality, but high more token usage + quality: str = "low" + # Max tokens to use for the API request, the maximum might be 3,000 but we don't know + max_tokens: int = 200 + client = OpenAI( + api_key=api_key, + max_retries=max_retries, + ) + logger = logging_config() + + class Config: + """Config class for the GPT4Vision model""" + + arbitary_types_allowed = True + + def process_img(self, img: str) -> str: + """Processes the image to be used for the API request""" + with open(img, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + + def __call__( + self, + img: Union[str, List[str]], + tasks: List[str], + ) -> GPT4VisionResponse: + """ + Calls the GPT-4 Vision API and returns the image url + + Parameters: + ----------- + img: Union[str, List[str]] + The image to be used for the API request + tasks: List[str] + The tasks to be used for the API request + + Returns: + -------- + answer: GPT4VisionResponse + The response from the API request + + Example: + -------- + >>> gpt4vision = GPT4Vision() + >>> img = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + >>> tasks = ["A painting of a dog"] + >>> answer = gpt4vision(img, tasks) + >>> print(answer) + + + """ + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + + # Image content + image_content = [ + {"type": "imavge_url", "image_url": img} + if img.startswith("http") + else {"type": "image", "data": img} + for img in img + ] + + messages = [ + { + "role": "user", + "content": image_content + [{"type": "text", "text": q} for q in tasks], + } + ] + + payload = { + "model": "gpt-4-vision-preview", + "messages": messages, + "max_tokens": self.max_tokens, + "detail": self.quality, + } + + for attempt in range(self.max_retries): + try: + response = requests.post( + "https://api.openai.com/v1/chat/completions", + headers=headers, + json=payload, + timeout=self.timeout_seconds, + ) + response.raise_for_status() + answer = response.json()["choices"][0]["message"]["content"]["text"] + return GPT4VisionResponse(answer=answer) + except requests.exceptions.HTTPError as error: + self.logger.error( + f"HTTP error: {error.response.status_code}, {error.response.text}" + ) + if error.response.status_code in [429, 500, 503]: + # Exponential backoff = 429(too many requesys) + # And 503 = (Service unavailable) errors + time.sleep(self.backoff_factor**attempt) + else: + break + + except requests.exceptions.RequestException as error: + self.logger.error(f"Request error: {error}") + time.sleep(self.backoff_factor**attempt) + except Exception as error: + self.logger.error( + f"Unexpected Error: {error} try optimizing your api key and try again" + ) + raise error from None + + raise TimeoutError("API Request timed out after multiple retries") + + def run(self, task: str, img: str) -> str: + """ + Runs the GPT-4 Vision API + + Parameters: + ----------- + task: str + The task to be used for the API request + img: str + The image to be used for the API request + + Returns: + -------- + out: str + The response from the API request + + Example: + -------- + >>> gpt4vision = GPT4Vision() + >>> task = "A painting of a dog" + >>> img = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + >>> answer = gpt4vision.run(task, img) + >>> print(answer) + """ + try: + response = self.client.chat.completions.create( + model=self.model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": f"{task}"}, + { + "type": "image_url", + "image_url": f"{img}", + }, + ], + } + ], + max_tokens=self.max_tokens, + ) + + out = response.choices[0].text + return out + except Exception as error: + print( + colored( + f"Error when calling GPT4Vision, Error: {error} Try optimizing your key, and try again", + "red", + ) + ) + + async def arun(self, task: str, img: str) -> str: + """ + Asynchronous run method for GPT-4 Vision + + Parameters: + ----------- + task: str + The task to be used for the API request + img: str + The image to be used for the API request + + Returns: + -------- + out: str + The response from the API request + + Example: + -------- + >>> gpt4vision = GPT4Vision() + >>> task = "A painting of a dog" + >>> img = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + >>> answer = await gpt4vision.arun(task, img) + >>> print(answer) + """ + try: + response = await self.client.chat.completions.create( + model=self.model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": f"{task}"}, + { + "type": "image_url", + "image_url": f"{img}", + }, + ], + } + ], + max_tokens=self.max_tokens, + ) + out = response.choices[0].text + return out + except Exception as error: + print( + colored( + f"Error when calling GPT4Vision, Error: {error} Try optimizing your key, and try again", + "red", + ) + ) diff --git a/swarms/structs/base.py b/swarms/structs/base.py index f33a204e..4208ba39 100644 --- a/swarms/structs/base.py +++ b/swarms/structs/base.py @@ -2,4 +2,4 @@ Base Structure for all Swarm Structures -""" \ No newline at end of file +""" diff --git a/swarms/utils/code_interpreter.py b/swarms/utils/code_interpreter.py index af6eb327..2448edc7 100644 --- a/swarms/utils/code_interpreter.py +++ b/swarms/utils/code_interpreter.py @@ -24,7 +24,7 @@ class SubprocessCodeInterpreter(BaseCodeInterpreter): """ SubprocessCodeinterpreter is a base class for code interpreters that run code in a subprocess. - + Attributes: start_cmd (str): The command to start the subprocess. Should be a string that can be split by spaces. process (subprocess.Popen): The subprocess that is running the code. diff --git a/tests/models/dalle3.py b/tests/models/dalle3.py new file mode 100644 index 00000000..ff1489ea --- /dev/null +++ b/tests/models/dalle3.py @@ -0,0 +1,374 @@ +import os +from unittest.mock import Mock + +import pytest +from openai import OpenAIError +from PIL import Image +from termcolor import colored + +from dalle3 import Dalle3 + + +# Mocking the OpenAI client to avoid making actual API calls during testing +@pytest.fixture +def mock_openai_client(): + return Mock() + + +@pytest.fixture +def dalle3(mock_openai_client): + return Dalle3(client=mock_openai_client) + + +def test_dalle3_call_success(dalle3, mock_openai_client): + # Arrange + task = "A painting of a dog" + expected_img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + mock_openai_client.images.generate.return_value = Mock(data=[Mock(url=expected_img_url)]) + + # Act + img_url = dalle3(task) + + # Assert + assert img_url == expected_img_url + mock_openai_client.images.generate.assert_called_once_with(prompt=task, n=4) + + +def test_dalle3_call_failure(dalle3, mock_openai_client, capsys): + # Arrange + task = "Invalid task" + expected_error_message = "Error running Dalle3: API Error" + + # Mocking OpenAIError + mock_openai_client.images.generate.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + + # Act and assert + with pytest.raises(OpenAIError) as excinfo: + dalle3(task) + + assert str(excinfo.value) == expected_error_message + mock_openai_client.images.generate.assert_called_once_with(prompt=task, n=4) + + # Ensure the error message is printed in red + captured = capsys.readouterr() + assert colored(expected_error_message, "red") in captured.out + + +def test_dalle3_create_variations_success(dalle3, mock_openai_client): + # Arrange + img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + expected_variation_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" + mock_openai_client.images.create_variation.return_value = Mock(data=[Mock(url=expected_variation_url)]) + + # Act + variation_img_url = dalle3.create_variations(img_url) + + # Assert + assert variation_img_url == expected_variation_url + mock_openai_client.images.create_variation.assert_called_once() + _, kwargs = mock_openai_client.images.create_variation.call_args + assert kwargs["img"] is not None + assert kwargs["n"] == 4 + assert kwargs["size"] == "1024x1024" + + +def test_dalle3_create_variations_failure(dalle3, mock_openai_client, capsys): + # Arrange + img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + expected_error_message = "Error running Dalle3: API Error" + + # Mocking OpenAIError + mock_openai_client.images.create_variation.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + + # Act and assert + with pytest.raises(OpenAIError) as excinfo: + dalle3.create_variations(img_url) + + assert str(excinfo.value) == expected_error_message + mock_openai_client.images.create_variation.assert_called_once() + + # Ensure the error message is printed in red + captured = capsys.readouterr() + assert colored(expected_error_message, "red") in captured.out + + +def test_dalle3_read_img(): + # Arrange + img_path = "test_image.png" + img = Image.new("RGB", (512, 512)) + + # Save the image temporarily + img.save(img_path) + + # Act + dalle3 = Dalle3() + img_loaded = dalle3.read_img(img_path) + + # Assert + assert isinstance(img_loaded, Image.Image) + + # Clean up + os.remove(img_path) + + +def test_dalle3_set_width_height(): + # Arrange + img = Image.new("RGB", (512, 512)) + width = 256 + height = 256 + + # Act + dalle3 = Dalle3() + img_resized = dalle3.set_width_height(img, width, height) + + # Assert + assert img_resized.size == (width, height) + + +def test_dalle3_convert_to_bytesio(): + # Arrange + img = Image.new("RGB", (512, 512)) + expected_format = "PNG" + + # Act + dalle3 = Dalle3() + img_bytes = dalle3.convert_to_bytesio(img, format=expected_format) + + # Assert + assert isinstance(img_bytes, bytes) + assert img_bytes.startswith(b"\x89PNG") + + +def test_dalle3_call_multiple_times(dalle3, mock_openai_client): + # Arrange + task = "A painting of a dog" + expected_img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + mock_openai_client.images.generate.return_value = Mock(data=[Mock(url=expected_img_url)]) + + # Act + img_url1 = dalle3(task) + img_url2 = dalle3(task) + + # Assert + assert img_url1 == expected_img_url + assert img_url2 == expected_img_url + assert mock_openai_client.images.generate.call_count == 2 + + +def test_dalle3_call_with_large_input(dalle3, mock_openai_client): + # Arrange + task = "A" * 2048 # Input longer than API's limit + expected_error_message = "Error running Dalle3: API Error" + mock_openai_client.images.generate.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + + # Act and assert + with pytest.raises(OpenAIError) as excinfo: + dalle3(task) + + assert str(excinfo.value) == expected_error_message + + +def test_dalle3_create_variations_with_invalid_image_url(dalle3, mock_openai_client): + # Arrange + img_url = "https://invalid-image-url.com" + expected_error_message = "Error running Dalle3: Invalid image URL" + + # Act and assert + with pytest.raises(ValueError) as excinfo: + dalle3.create_variations(img_url) + + assert str(excinfo.value) == expected_error_message + + +def test_dalle3_set_width_height_invalid_dimensions(dalle3): + # Arrange + img = dalle3.read_img("test_image.png") + width = 0 + height = -1 + + # Act and assert + with pytest.raises(ValueError): + dalle3.set_width_height(img, width, height) + + +def test_dalle3_convert_to_bytesio_invalid_format(dalle3): + # Arrange + img = dalle3.read_img("test_image.png") + invalid_format = "invalid_format" + + # Act and assert + with pytest.raises(ValueError): + dalle3.convert_to_bytesio(img, format=invalid_format) + + +def test_dalle3_call_with_retry(dalle3, mock_openai_client): + # Arrange + task = "A painting of a dog" + expected_img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + + # Simulate a retry scenario + mock_openai_client.images.generate.side_effect = [ + OpenAIError("Temporary error", http_status=500, error="Internal Server Error"), + Mock(data=[Mock(url=expected_img_url)]), + ] + + # Act + img_url = dalle3(task) + + # Assert + assert img_url == expected_img_url + assert mock_openai_client.images.generate.call_count == 2 + + +def test_dalle3_create_variations_with_retry(dalle3, mock_openai_client): + # Arrange + img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + expected_variation_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" + + # Simulate a retry scenario + mock_openai_client.images.create_variation.side_effect = [ + OpenAIError("Temporary error", http_status=500, error="Internal Server Error"), + Mock(data=[Mock(url=expected_variation_url)]), + ] + + # Act + variation_img_url = dalle3.create_variations(img_url) + + # Assert + assert variation_img_url == expected_variation_url + assert mock_openai_client.images.create_variation.call_count == 2 + + +def test_dalle3_call_exception_logging(dalle3, mock_openai_client, capsys): + # Arrange + task = "A painting of a dog" + expected_error_message = "Error running Dalle3: API Error" + + # Mocking OpenAIError + mock_openai_client.images.generate.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + + # Act + with pytest.raises(OpenAIError): + dalle3(task) + + # Assert that the error message is logged + captured = capsys.readouterr() + assert expected_error_message in captured.err + + +def test_dalle3_create_variations_exception_logging(dalle3, mock_openai_client, capsys): + # Arrange + img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + expected_error_message = "Error running Dalle3: API Error" + + # Mocking OpenAIError + mock_openai_client.images.create_variation.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + + # Act + with pytest.raises(OpenAIError): + dalle3.create_variations(img_url) + + # Assert that the error message is logged + captured = capsys.readouterr() + assert expected_error_message in captured.err + + +def test_dalle3_read_img_invalid_path(dalle3): + # Arrange + invalid_img_path = "invalid_image_path.png" + + # Act and assert + with pytest.raises(FileNotFoundError): + dalle3.read_img(invalid_img_path) + + +def test_dalle3_call_no_api_key(): + # Arrange + task = "A painting of a dog" + dalle3 = Dalle3(api_key=None) + expected_error_message = "Error running Dalle3: API Key is missing" + + # Act and assert + with pytest.raises(ValueError) as excinfo: + dalle3(task) + + assert str(excinfo.value) == expected_error_message + + +def test_dalle3_create_variations_no_api_key(): + # Arrange + img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + dalle3 = Dalle3(api_key=None) + expected_error_message = "Error running Dalle3: API Key is missing" + + # Act and assert + with pytest.raises(ValueError) as excinfo: + dalle3.create_variations(img_url) + + assert str(excinfo.value) == expected_error_message + + +def test_dalle3_call_with_retry_max_retries_exceeded(dalle3, mock_openai_client): + # Arrange + task = "A painting of a dog" + + # Simulate max retries exceeded + mock_openai_client.images.generate.side_effect = OpenAIError("Temporary error", http_status=500, error="Internal Server Error") + + # Act and assert + with pytest.raises(OpenAIError) as excinfo: + dalle3(task) + + assert "Retry limit exceeded" in str(excinfo.value) + + +def test_dalle3_create_variations_with_retry_max_retries_exceeded(dalle3, mock_openai_client): + # Arrange + img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + + # Simulate max retries exceeded + mock_openai_client.images.create_variation.side_effect = OpenAIError("Temporary error", http_status=500, error="Internal Server Error") + + # Act and assert + with pytest.raises(OpenAIError) as excinfo: + dalle3.create_variations(img_url) + + assert "Retry limit exceeded" in str(excinfo.value) + + +def test_dalle3_call_retry_with_success(dalle3, mock_openai_client): + # Arrange + task = "A painting of a dog" + expected_img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + + # Simulate success after a retry + mock_openai_client.images.generate.side_effect = [ + OpenAIError("Temporary error", http_status=500, error="Internal Server Error"), + Mock(data=[Mock(url=expected_img_url)]), + ] + + # Act + img_url = dalle3(task) + + # Assert + assert img_url == expected_img_url + assert mock_openai_client.images.generate.call_count == 2 + + +def test_dalle3_create_variations_retry_with_success(dalle3, mock_openai_client): + # Arrange + img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + expected_variation_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" + + # Simulate success after a retry + mock_openai_client.images.create_variation.side_effect = [ + OpenAIError("Temporary error", http_status=500, error="Internal Server Error"), + Mock(data=[Mock(url=expected_variation_url)]), + ] + + # Act + variation_img_url = dalle3.create_variations(img_url) + + # Assert + assert variation_img_url == expected_variation_url + assert mock_openai_client.images.create_variation.call_count == 2 diff --git a/tests/models/gpt4v.py b/tests/models/gpt4v.py new file mode 100644 index 00000000..40ccc7f5 --- /dev/null +++ b/tests/models/gpt4v.py @@ -0,0 +1,321 @@ +import logging +import os +from unittest.mock import Mock + +import pytest +from dotenv import load_dotenv +from requests.exceptions import ConnectionError, HTTPError, RequestException, Timeout + +from swarms.models.gpt4v import GPT4Vision, GPT4VisionResponse + +load_dotenv + +api_key = os.getenv("OPENAI_API_KEY") + +# Mock the OpenAI client +@pytest.fixture +def mock_openai_client(): + return Mock() + +@pytest.fixture +def gpt4vision(mock_openai_client): + return GPT4Vision(client=mock_openai_client) + +def test_gpt4vision_default_values(): + # Arrange and Act + gpt4vision = GPT4Vision() + + # Assert + assert gpt4vision.max_retries == 3 + assert gpt4vision.model == "gpt-4-vision-preview" + assert gpt4vision.backoff_factor == 2.0 + assert gpt4vision.timeout_seconds == 10 + assert gpt4vision.api_key is None + assert gpt4vision.quality == "low" + assert gpt4vision.max_tokens == 200 + +def test_gpt4vision_api_key_from_env_variable(): + # Arrange + api_key = os.environ["OPENAI_API_KEY"] + + # Act + gpt4vision = GPT4Vision() + + # Assert + assert gpt4vision.api_key == api_key + +def test_gpt4vision_set_api_key(): + # Arrange + gpt4vision = GPT4Vision(api_key=api_key) + + # Assert + assert gpt4vision.api_key == api_key + +def test_gpt4vision_invalid_max_retries(): + # Arrange and Act + with pytest.raises(ValueError): + GPT4Vision(max_retries=-1) + +def test_gpt4vision_invalid_backoff_factor(): + # Arrange and Act + with pytest.raises(ValueError): + GPT4Vision(backoff_factor=-1) + +def test_gpt4vision_invalid_timeout_seconds(): + # Arrange and Act + with pytest.raises(ValueError): + GPT4Vision(timeout_seconds=-1) + +def test_gpt4vision_invalid_max_tokens(): + # Arrange and Act + with pytest.raises(ValueError): + GPT4Vision(max_tokens=-1) + +def test_gpt4vision_logger_initialized(): + # Arrange + gpt4vision = GPT4Vision() + + # Assert + assert isinstance(gpt4vision.logger, logging.Logger) + +def test_gpt4vision_process_img_nonexistent_file(): + # Arrange + gpt4vision = GPT4Vision() + img_path = "nonexistent_image.jpg" + + # Act and Assert + with pytest.raises(FileNotFoundError): + gpt4vision.process_img(img_path) + +def test_gpt4vision_call_single_task_single_image_no_openai_client(gpt4vision): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + # Act and Assert + with pytest.raises(AttributeError): + gpt4vision(img_url, [task]) + +def test_gpt4vision_call_single_task_single_image_empty_response(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + mock_openai_client.chat.completions.create.return_value.choices = [] + + # Act + response = gpt4vision(img_url, [task]) + + # Assert + assert response.answer == "" + mock_openai_client.chat.completions.create.assert_called_once() + +def test_gpt4vision_call_multiple_tasks_single_image_empty_responses(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + tasks = ["Describe this image.", "What's in this picture?"] + + mock_openai_client.chat.completions.create.return_value.choices = [] + + # Act + responses = gpt4vision(img_url, tasks) + + # Assert + assert all(response.answer == "" for response in responses) + assert mock_openai_client.chat.completions.create.call_count == 1 # Should be called only once + +def test_gpt4vision_call_single_task_single_image_timeout(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + mock_openai_client.chat.completions.create.side_effect = Timeout("Request timed out") + + # Act and Assert + with pytest.raises(Timeout): + gpt4vision(img_url, [task]) + +def test_gpt4vision_call_retry_with_success_after_timeout(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + # Simulate success after a timeout and retry + mock_openai_client.chat.completions.create.side_effect = [ + Timeout("Request timed out"), + {"choices": [{"message": {"content": {"text": "A description of the image."}}}],} + ] + + # Act + response = gpt4vision(img_url, [task]) + + # Assert + assert response.answer == "A description of the image." + assert mock_openai_client.chat.completions.create.call_count == 2 # Should be called twice + + +def test_gpt4vision_process_img(): + # Arrange + img_path = "test_image.jpg" + gpt4vision = GPT4Vision() + + # Act + img_data = gpt4vision.process_img(img_path) + + # Assert + assert img_data.startswith("/9j/") # Base64-encoded image data + + +def test_gpt4vision_call_single_task_single_image(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + expected_response = GPT4VisionResponse(answer="A description of the image.") + + mock_openai_client.chat.completions.create.return_value.choices[0].text = expected_response.answer + + # Act + response = gpt4vision(img_url, [task]) + + # Assert + assert response == expected_response + mock_openai_client.chat.completions.create.assert_called_once() + + +def test_gpt4vision_call_single_task_multiple_images(gpt4vision, mock_openai_client): + # Arrange + img_urls = ["https://example.com/image1.jpg", "https://example.com/image2.jpg"] + task = "Describe these images." + + expected_response = GPT4VisionResponse(answer="Descriptions of the images.") + + mock_openai_client.chat.completions.create.return_value.choices[0].text = expected_response.answer + + # Act + response = gpt4vision(img_urls, [task]) + + # Assert + assert response == expected_response + mock_openai_client.chat.completions.create.assert_called_once() + + +def test_gpt4vision_call_multiple_tasks_single_image(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + tasks = ["Describe this image.", "What's in this picture?"] + + expected_responses = [ + GPT4VisionResponse(answer="A description of the image."), + GPT4VisionResponse(answer="It contains various objects."), + ] + + def create_mock_response(response): + return {"choices": [{"message": {"content": {"text": response.answer}}}]} + + mock_openai_client.chat.completions.create.side_effect = [create_mock_response(response) for response in expected_responses] + + # Act + responses = gpt4vision(img_url, tasks) + + # Assert + assert responses == expected_responses + assert mock_openai_client.chat.completions.create.call_count == 1 # Should be called only once + def test_gpt4vision_call_multiple_tasks_single_image(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + tasks = ["Describe this image.", "What's in this picture?"] + + expected_responses = [ + GPT4VisionResponse(answer="A description of the image."), + GPT4VisionResponse(answer="It contains various objects."), + ] + + mock_openai_client.chat.completions.create.side_effect = [ + {"choices": [{"message": {"content": {"text": expected_responses[i].answer}}}] } for i in range(len(expected_responses)) + ] + + # Act + responses = gpt4vision(img_url, tasks) + + # Assert + assert responses == expected_responses + assert mock_openai_client.chat.completions.create.call_count == 1 # Should be called only once + + +def test_gpt4vision_call_multiple_tasks_multiple_images(gpt4vision, mock_openai_client): + # Arrange + img_urls = ["https://images.unsplash.com/photo-1694734479857-626882b6db37?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D", "https://images.unsplash.com/photo-1694734479898-6ac4633158ac?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D"] + tasks = ["Describe these images.", "What's in these pictures?"] + + expected_responses = [ + GPT4VisionResponse(answer="Descriptions of the images."), + GPT4VisionResponse(answer="They contain various objects.") + ] + + mock_openai_client.chat.completions.create.side_effect = [ + {"choices": [{"message": {"content": {"text": response.answer}}}] } for response in expected_responses + ] + + # Act + responses = gpt4vision(img_urls, tasks) + + + # Assert + assert responses == expected_responses + assert mock_openai_client.chat.completions.create.call_count == 1 # Should be called only once + + +def test_gpt4vision_call_http_error(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + mock_openai_client.chat.completions.create.side_effect = HTTPError("HTTP Error") + + # Act and Assert + with pytest.raises(HTTPError): + gpt4vision(img_url, [task]) + + +def test_gpt4vision_call_request_error(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + mock_openai_client.chat.completions.create.side_effect = RequestException("Request Error") + + # Act and Assert + with pytest.raises(RequestException): + gpt4vision(img_url, [task]) + + +def test_gpt4vision_call_connection_error(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + mock_openai_client.chat.completions.create.side_effect = ConnectionError("Connection Error") + + # Act and Assert + with pytest.raises(ConnectionError): + gpt4vision(img_url, [task]) + + +def test_gpt4vision_call_retry_with_success(gpt4vision, mock_openai_client): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + task = "Describe this image." + + # Simulate success after a retry + mock_openai_client.chat.completions.create.side_effect = [ + RequestException("Temporary error"), + {"choices": [{"text": "A description of the image."}]} # fixed dictionary syntax + ] + + # Act + response = gpt4vision(img_url, [task]) + + # Assert + assert response.answer == "A description of the image." + assert mock_openai_client.chat.completions.create.call_count == 2 # Should be called twice diff --git a/tests/structs/sequential_workflow.py b/tests/structs/sequential_workflow.py index 64b51f28..7bd3e4a4 100644 --- a/tests/structs/sequential_workflow.py +++ b/tests/structs/sequential_workflow.py @@ -12,7 +12,6 @@ from swarms.structs.sequential_workflow import SequentialWorkflow, Task os.environ["OPENAI_API_KEY"] = "mocked_api_key" - # Mock OpenAIChat class for testing class MockOpenAIChat: def __init__(self, *args, **kwargs): @@ -21,6 +20,7 @@ class MockOpenAIChat: def run(self, *args, **kwargs): return "Mocked result" + # Mock Flow class for testing class MockFlow: def __init__(self, *args, **kwargs): @@ -29,6 +29,7 @@ class MockFlow: def run(self, *args, **kwargs): return "Mocked result" + # Mock SequentialWorkflow class for testing class MockSequentialWorkflow: def __init__(self, *args, **kwargs): @@ -40,6 +41,7 @@ class MockSequentialWorkflow: def run(self): pass + # Test Task class def test_task_initialization(): description = "Sample Task" @@ -48,6 +50,7 @@ def test_task_initialization(): assert task.description == description assert task.flow == flow + def test_task_execute(): description = "Sample Task" flow = MockOpenAIChat() @@ -55,6 +58,7 @@ def test_task_execute(): task.execute() assert task.result == "Mocked result" + # Test SequentialWorkflow class def test_sequential_workflow_initialization(): workflow = SequentialWorkflow() @@ -66,6 +70,7 @@ def test_sequential_workflow_initialization(): assert workflow.restore_state_filepath == None assert workflow.dashboard == False + def test_sequential_workflow_add_task(): workflow = SequentialWorkflow() task_description = "Sample Task" @@ -75,6 +80,7 @@ def test_sequential_workflow_add_task(): assert workflow.tasks[0].description == task_description assert workflow.tasks[0].flow == task_flow + def test_sequential_workflow_reset_workflow(): workflow = SequentialWorkflow() task_description = "Sample Task" @@ -83,6 +89,7 @@ def test_sequential_workflow_reset_workflow(): workflow.reset_workflow() assert workflow.tasks[0].result == None + def test_sequential_workflow_get_task_results(): workflow = SequentialWorkflow() task_description = "Sample Task" @@ -94,6 +101,7 @@ def test_sequential_workflow_get_task_results(): assert task_description in results assert results[task_description] == "Mocked result" + def test_sequential_workflow_remove_task(): workflow = SequentialWorkflow() task1_description = "Task 1" @@ -106,6 +114,7 @@ def test_sequential_workflow_remove_task(): assert len(workflow.tasks) == 1 assert workflow.tasks[0].description == task2_description + def test_sequential_workflow_update_task(): workflow = SequentialWorkflow() task_description = "Sample Task" @@ -114,6 +123,7 @@ def test_sequential_workflow_update_task(): workflow.update_task(task_description, max_tokens=1000) assert workflow.tasks[0].kwargs["max_tokens"] == 1000 + def test_sequential_workflow_save_workflow_state(): workflow = SequentialWorkflow() task_description = "Sample Task" @@ -123,6 +133,7 @@ def test_sequential_workflow_save_workflow_state(): assert os.path.exists("test_state.json") os.remove("test_state.json") + def test_sequential_workflow_load_workflow_state(): workflow = SequentialWorkflow() task_description = "Sample Task" @@ -134,6 +145,7 @@ def test_sequential_workflow_load_workflow_state(): assert workflow.tasks[0].description == task_description os.remove("test_state.json") + def test_sequential_workflow_run(): workflow = SequentialWorkflow() task_description = "Sample Task" @@ -142,18 +154,21 @@ def test_sequential_workflow_run(): workflow.run() assert workflow.tasks[0].result == "Mocked result" + def test_sequential_workflow_workflow_bootup(capfd): workflow = SequentialWorkflow() workflow.workflow_bootup() out, _ = capfd.readouterr() assert "Sequential Workflow Initializing..." in out + def test_sequential_workflow_workflow_dashboard(capfd): workflow = SequentialWorkflow() workflow.workflow_dashboard() out, _ = capfd.readouterr() assert "Sequential Workflow Dashboard" in out + # Mock Flow class for async testing class MockAsyncFlow: def __init__(self, *args, **kwargs): @@ -162,6 +177,7 @@ class MockAsyncFlow: async def arun(self, *args, **kwargs): return "Mocked result" + # Test async execution in SequentialWorkflow @pytest.mark.asyncio async def test_sequential_workflow_arun(): @@ -173,23 +189,24 @@ async def test_sequential_workflow_arun(): assert workflow.tasks[0].result == "Mocked result" - - def test_real_world_usage_with_openai_key(): # Initialize the language model llm = OpenAIChat() assert isinstance(llm, OpenAIChat) + def test_real_world_usage_with_flow_and_openai_key(): # Initialize a flow with the language model flow = Flow(llm=OpenAIChat()) assert isinstance(flow, Flow) + def test_real_world_usage_with_sequential_workflow(): # Initialize a sequential workflow workflow = SequentialWorkflow() assert isinstance(workflow, SequentialWorkflow) + def test_real_world_usage_add_tasks(): # Create a sequential workflow and add tasks workflow = SequentialWorkflow() @@ -203,6 +220,7 @@ def test_real_world_usage_add_tasks(): assert workflow.tasks[0].description == task1_description assert workflow.tasks[1].description == task2_description + def test_real_world_usage_run_workflow(): # Create a sequential workflow, add a task, and run the workflow workflow = SequentialWorkflow() @@ -212,6 +230,7 @@ def test_real_world_usage_run_workflow(): workflow.run() assert workflow.tasks[0].result is not None + def test_real_world_usage_dashboard_display(): # Create a sequential workflow, add tasks, and display the dashboard workflow = SequentialWorkflow() @@ -225,6 +244,7 @@ def test_real_world_usage_dashboard_display(): workflow.workflow_dashboard() mock_print.assert_called() + def test_real_world_usage_async_execution(): # Create a sequential workflow, add an async task, and run the workflow asynchronously workflow = SequentialWorkflow() @@ -238,6 +258,7 @@ def test_real_world_usage_async_execution(): asyncio.run(async_run_workflow()) assert workflow.tasks[0].result is not None + def test_real_world_usage_multiple_loops(): # Create a sequential workflow with multiple loops, add a task, and run the workflow workflow = SequentialWorkflow(max_loops=3) @@ -247,6 +268,7 @@ def test_real_world_usage_multiple_loops(): workflow.run() assert workflow.tasks[0].result is not None + def test_real_world_usage_autosave_state(): # Create a sequential workflow with autosave, add a task, run the workflow, and check if state is saved workflow = SequentialWorkflow(autosave=True) @@ -258,6 +280,7 @@ def test_real_world_usage_autosave_state(): assert os.path.exists("sequential_workflow_state.json") os.remove("sequential_workflow_state.json") + def test_real_world_usage_load_state(): # Create a sequential workflow, add a task, save state, load state, and run the workflow workflow = SequentialWorkflow() @@ -271,6 +294,7 @@ def test_real_world_usage_load_state(): assert workflow.tasks[0].result is not None os.remove("test_state.json") + def test_real_world_usage_update_task_args(): # Create a sequential workflow, add a task, and update task arguments workflow = SequentialWorkflow() @@ -280,6 +304,7 @@ def test_real_world_usage_update_task_args(): workflow.update_task(task_description, max_tokens=1000) assert workflow.tasks[0].kwargs["max_tokens"] == 1000 + def test_real_world_usage_remove_task(): # Create a sequential workflow, add tasks, remove a task, and run the workflow workflow = SequentialWorkflow() @@ -294,13 +319,15 @@ def test_real_world_usage_remove_task(): assert len(workflow.tasks) == 1 assert workflow.tasks[0].description == task2_description + def test_real_world_usage_with_environment_variables(): # Ensure that the OpenAI API key is set using environment variables assert "OPENAI_API_KEY" in os.environ assert os.environ["OPENAI_API_KEY"] == "mocked_api_key" del os.environ["OPENAI_API_KEY"] # Clean up after the test + def test_real_world_usage_no_openai_key(): # Ensure that an exception is raised when the OpenAI API key is not set with pytest.raises(ValueError): - llm = OpenAIChat() # API key not provided, should raise an exception \ No newline at end of file + llm = OpenAIChat() # API key not provided, should raise an exception From b7aa21f92b26c9442720ddafad1198d5021f2cf6 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 16:31:32 -0500 Subject: [PATCH 23/32] docs --- docs/swarms/models/dalle3.md | 261 +++++++++++++++++++++++++++++++++++ mkdocs.yml | 2 + 2 files changed, 263 insertions(+) create mode 100644 docs/swarms/models/dalle3.md diff --git a/docs/swarms/models/dalle3.md b/docs/swarms/models/dalle3.md new file mode 100644 index 00000000..ff12b130 --- /dev/null +++ b/docs/swarms/models/dalle3.md @@ -0,0 +1,261 @@ +# `Dalle3` Documentation + +## Table of Contents + +1. [Introduction](#introduction) +2. [Installation](#installation) +3. [Quick Start](#quick-start) +4. [Dalle3 Class](#dalle3-class) + - [Attributes](#attributes) + - [Methods](#methods) +5. [Usage Examples](#usage-examples) +6. [Error Handling](#error-handling) +7. [Advanced Usage](#advanced-usage) +8. [References](#references) + +--- + +## Introduction + +The Dalle3 library is a Python module that provides an easy-to-use interface for generating images from text descriptions using the DALL·E 3 model by OpenAI. DALL·E 3 is a powerful language model capable of converting textual prompts into images. This documentation will guide you through the installation, setup, and usage of the Dalle3 library. + +--- + +## Installation + +To install the Dalle3 library, you can use pip: + +```bash +pip install dalle3 +``` + +--- + +## Quick Start + +Let's get started with a quick example of using the Dalle3 library to generate an image from a text prompt: + +```python +from swarms.models.dalle3 import Dalle3 + +# Create an instance of the Dalle3 class +dalle = Dalle3() + +# Define a text prompt +task = "A painting of a dog" + +# Generate an image from the text prompt +image_url = dalle3(task) + +# Print the generated image URL +print(image_url) +``` + +This example demonstrates the basic usage of the Dalle3 library to convert a text prompt into an image. The generated image URL will be printed to the console. + +--- + +## Dalle3 Class + +The Dalle3 library provides a `Dalle3` class that allows you to interact with the DALL·E 3 model. This class has several attributes and methods for generating images from text prompts. + +### Attributes + +- `model` (str): The name of the DALL·E 3 model. Default: "dall-e-3". +- `img` (str): The image URL generated by the Dalle3 API. +- `size` (str): The size of the generated image. Default: "1024x1024". +- `max_retries` (int): The maximum number of API request retries. Default: 3. +- `quality` (str): The quality of the generated image. Default: "standard". +- `n` (int): The number of variations to create. Default: 4. + +### Methods + +#### `__call__(self, task: str) -> Dalle3` + +This method makes a call to the Dalle3 API and returns the image URL generated from the provided text prompt. + +Parameters: +- `task` (str): The text prompt to be converted to an image. + +Returns: +- `Dalle3`: An instance of the Dalle3 class with the image URL generated by the Dalle3 API. + +#### `create_variations(self, img: str)` + +This method creates variations of an image using the Dalle3 API. + +Parameters: +- `img` (str): The image to be used for the API request. + +Returns: +- `img` (str): The image URL of the generated variations. + +--- + +## Usage Examples + +### Example 1: Basic Image Generation + +```python +from swarms.models.dalle3 import Dalle3 + +# Create an instance of the Dalle3 class +dalle3 = Dalle3() + +# Define a text prompt +task = "A painting of a dog" + +# Generate an image from the text prompt +image_url = dalle3(task) + +# Print the generated image URL +print(image_url) +``` + +### Example 2: Creating Image Variations + +```python +from swarms.models.dalle3 import Dalle3 + +# Create an instance of the Dalle3 class +dalle3 = Dalle3() + +# Define the URL of an existing image +img_url = "https://images.unsplash.com/photo-1694734479898-6ac4633158ac?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D + +# Create variations of the image +variations_url = dalle3.create_variations(img_url) + +# Print the URLs of the generated variations +print(variations_url) +``` + +Certainly! Here are additional examples that cover various edge cases and methods of the `Dalle3` class in the Dalle3 library: + +### Example 3: Customizing Image Size + +You can customize the size of the generated image by specifying the `size` parameter when creating an instance of the `Dalle3` class. Here's how to generate a smaller image: + +```python +from swarms.models.dalle3 import Dalle3 + +# Create an instance of the Dalle3 class with a custom image size +dalle3 = Dalle3(size="512x512") + +# Define a text prompt +task = "A small painting of a cat" + +# Generate a smaller image from the text prompt +image_url = dalle3(task) + +# Print the generated image URL +print(image_url) +``` + +### Example 4: Adjusting Retry Limit + +You can adjust the maximum number of API request retries using the `max_retries` parameter. Here's how to increase the retry limit: + +```python +from swarms.models.dalle3 import Dalle3 + +# Create an instance of the Dalle3 class with a higher retry limit +dalle3 = Dalle3(max_retries=5) + +# Define a text prompt +task = "An image of a landscape" + +# Generate an image with a higher retry limit +image_url = dalle3(task) + +# Print the generated image URL +print(image_url) +``` + +### Example 5: Generating Image Variations + +To create variations of an existing image, you can use the `create_variations` method. Here's an example: + +```python +from swarms.models.dalle3 import Dalle3 + +# Create an instance of the Dalle3 class +dalle3 = Dalle3() + +# Define the URL of an existing image +img_url = "https://images.unsplash.com/photo-1677290043066-12eccd944004?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + +# Create variations of the image +variations_url = dalle3.create_variations(img_url) + +# Print the URLs of the generated variations +print(variations_url) +``` + +### Example 6: Handling API Errors + +The Dalle3 library provides error handling for API-related issues. Here's how to handle and display API errors: + +```python +from swarms.models.dalle3 import Dalle3 + +# Create an instance of the Dalle3 class +dalle3 = Dalle3() + +# Define a text prompt +task = "Invalid prompt that may cause an API error" + +try: + # Attempt to generate an image with an invalid prompt + image_url = dalle3(task) + print(image_url) +except Exception as e: + print(f"Error occurred: {str(e)}") +``` + +### Example 7: Customizing Image Quality + +You can customize the quality of the generated image by specifying the `quality` parameter. Here's how to generate a high-quality image: + +```python +from swarms.models.dalle3 import Dalle3 + +# Create an instance of the Dalle3 class with high quality +dalle3 = Dalle3(quality="high") + +# Define a text prompt +task = "A high-quality image of a sunset" + +# Generate a high-quality image from the text prompt +image_url = dalle3(task) + +# Print the generated image URL +print(image_url) +``` + + +--- + +## Error Handling + +The Dalle3 library provides error handling for API-related issues. If an error occurs during API communication, the library will handle it and provide detailed error messages. Make sure to handle exceptions appropriately in your code. + +--- + +## Advanced Usage + +For advanced usage and customization of the Dalle3 library, you can explore the attributes and methods of the `Dalle3` class. Adjusting parameters such as `size`, `max_retries`, and `quality` allows you to fine-tune the image generation process to your specific needs. + +--- + +## References + +For more information about the DALL·E 3 model and the Dalle3 library, you can refer to the official OpenAI documentation and resources. + +- [OpenAI API Documentation](https://beta.openai.com/docs/) +- [DALL·E 3 Model Information](https://openai.com/research/dall-e-3) +- [Dalle3 GitHub Repository](https://github.com/openai/dall-e-3) + +--- + +This concludes the documentation for the Dalle3 library. You can now use the library to generate images from text prompts and explore its advanced features for various applications. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 58430091..7413e809 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -92,6 +92,8 @@ nav: - BingChat: "swarms/models/bingchat.md" - Kosmos: "swarms/models/kosmos.md" - Nougat: "swarms/models/nougat.md" + - Dalle3: "swarms/models/dalle3.md" + - GPT4V: "swarms/models/gpt4v.md" - LayoutLMDocumentQA: "swarms/models/layoutlm_document_qa.md" - DistilWhisperModel: "swarms/models/distilled_whisperx.md" - swarms.structs: From c7d128d8601a007dc89bfc0589d26da4138606c2 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 16:32:27 -0500 Subject: [PATCH 24/32] docs for gpt4v --- docs/swarms/models/gpt4v.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/swarms/models/gpt4v.md b/docs/swarms/models/gpt4v.md index 2af4348b..3fe3d81c 100644 --- a/docs/swarms/models/gpt4v.md +++ b/docs/swarms/models/gpt4v.md @@ -1,4 +1,4 @@ -# GPT4Vision Documentation +# `GPT4Vision` Documentation ## Table of Contents - [Overview](#overview) From 4fb38c1c62d39cfdfe8c0d45f51ae9523eadeee5 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 16:49:10 -0500 Subject: [PATCH 25/32] dalle3 --- docs/swarms/models/dalle3.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/swarms/models/dalle3.md b/docs/swarms/models/dalle3.md index ff12b130..346489c7 100644 --- a/docs/swarms/models/dalle3.md +++ b/docs/swarms/models/dalle3.md @@ -23,10 +23,10 @@ The Dalle3 library is a Python module that provides an easy-to-use interface for ## Installation -To install the Dalle3 library, you can use pip: +To use the Dalle3 model, you must first install swarms: ```bash -pip install dalle3 +pip install swarms ``` --- From a70a2b05b5c7ba27bdc27a00678e097a8432964a Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 17:25:05 -0500 Subject: [PATCH 26/32] Dependencies clean up --- README.md | 30 +++++++++++++++++++++++------- pyproject.toml | 5 +---- swarms/models/__init__.py | 4 ++++ swarms/models/dalle3.py | 19 +++++++++---------- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index f94221d4..a80a307a 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ We have a small gallery of examples to run here, [for more check out the docs to ### `Flow` Example - The `Flow` is a superior iteratioin of the `LLMChain` from Langchain, our intent with `Flow` is to create the most reliable loop structure that gives the agents their "autonomy" through 3 main methods of interaction, one through user specified loops, then dynamic where the agent parses a token, and or an interactive human input verison, or a mix of all 3. + ```python from swarms.models import OpenAIChat @@ -47,22 +48,37 @@ from swarms.structs import Flow api_key = "" - -# Initialize the language model, -# This model can be swapped out with Anthropic, ETC, Huggingface Models like Mistral, ETC +# Initialize the language model, this model can be swapped out with Anthropic, ETC, Huggingface Models like Mistral, ETC llm = OpenAIChat( + # model_name="gpt-4" openai_api_key=api_key, temperature=0.5, + # max_tokens=100, ) -# Initialize the flow +## Initialize the workflow flow = Flow( llm=llm, - max_loops=5, + max_loops=2, + dashboard=True, + # stopping_condition=None, # You can define a stopping condition as needed. + # loop_interval=1, + # retry_attempts=3, + # retry_interval=1, + # interactive=False, # Set to 'True' for interactive mode. + # dynamic_temperature=False, # Set to 'True' for dynamic temperature handling. ) -out = flow.run("Generate a 10,000 word blog, say Stop when done") -print(out) +# out = flow.load_state("flow_state.json") +# temp = flow.dynamic_temperature() +# filter = flow.add_response_filter("Trump") +out = flow.run("Generate a 10,000 word blog on health and wellness.") +# out = flow.validate_response(out) +# out = flow.analyze_feedback(out) +# out = flow.print_history_and_memory() +# # out = flow.save_state("flow_state.json") +# print(out) + ``` diff --git a/pyproject.toml b/pyproject.toml index e3a29e78..d8a561bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "swarms" -version = "1.9.5" +version = "1.9.6" description = "Swarms - Pytorch" license = "MIT" authors = ["Kye Gomez "] @@ -28,7 +28,6 @@ openai = "*" langchain = "*" asyncio = "*" nest_asyncio = "*" -pegasusx = "*" einops = "*" google-generativeai = "*" torch = "*" @@ -48,10 +47,8 @@ beautifulsoup4 = "*" huggingface-hub = "*" pydantic = "*" tenacity = "*" -redis = "*" Pillow = "*" chromadb = "*" -agent-protocol = "*" open-interpreter = "*" tabulate = "*" termcolor = "*" diff --git a/swarms/models/__init__.py b/swarms/models/__init__.py index a0bec07f..b2a2b433 100644 --- a/swarms/models/__init__.py +++ b/swarms/models/__init__.py @@ -16,6 +16,8 @@ from swarms.models.kosmos_two import Kosmos from swarms.models.vilt import Vilt from swarms.models.nougat import Nougat from swarms.models.layoutlm_document_qa import LayoutLMDocumentQA +from swarms.models.gpt4v import GPT4Vision +from swarms.models.dalle3 import Dalle3 # from swarms.models.distilled_whisperx import DistilWhisperModel @@ -43,4 +45,6 @@ __all__ = [ "HuggingfaceLLM", "MPT7B", "WizardLLMStoryTeller", + "GPT4Vision", + "Dalle3", ] diff --git a/swarms/models/dalle3.py b/swarms/models/dalle3.py index f22b11e0..73edf502 100644 --- a/swarms/models/dalle3.py +++ b/swarms/models/dalle3.py @@ -1,15 +1,14 @@ -import openai import logging import os from dataclasses import dataclass -from functools import lru_cache -from termcolor import colored -from openai import OpenAI -from dotenv import load_dotenv -from pydantic import BaseModel, validator -from PIL import Image from io import BytesIO +import openai +from dotenv import load_dotenv +from openai import OpenAI +from PIL import Image +from pydantic import validator +from termcolor import colored load_dotenv() @@ -111,10 +110,10 @@ class Dalle3: try: # Making a call to the the Dalle3 API response = self.client.images.generate( - # model=self.model, + model=self.model, prompt=task, - # size=self.size, - # quality=self.quality, + size=self.size, + quality=self.quality, n=self.n, ) # Extracting the image url from the response From 336bffea19feff85db6e6b9035d687cdc09a4b0c Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 17:43:50 -0500 Subject: [PATCH 27/32] playground + flow docs fix --- docs/swarms/structs/flow.md | 11 ++- .../agents/simple_agent.py | 0 playground/models/multitemp.py | 56 ------------- playground/models/openai_model.py | 4 +- playground/structs/flow.py | 35 ++++++++ playground/structs/sequential_workflow.py | 31 +++++++ playground/swarms/godmode.py | 39 ++------- playground/swarms/groupchat.py | 84 ++++++++----------- 8 files changed, 120 insertions(+), 140 deletions(-) rename simple_agent.py => playground/agents/simple_agent.py (100%) delete mode 100644 playground/models/multitemp.py create mode 100644 playground/structs/sequential_workflow.py diff --git a/docs/swarms/structs/flow.md b/docs/swarms/structs/flow.md index 9300c632..13f0541c 100644 --- a/docs/swarms/structs/flow.md +++ b/docs/swarms/structs/flow.md @@ -108,8 +108,13 @@ Here are three usage examples: ```python from swarms.structs import Flow +# Select any Language model from the models folder +from swarms.models import Mistral, OpenAIChat -flow = Flow(llm=my_language_model, max_loops=5) +llm = Mistral() +# llm = OpenAIChat() + +flow = Flow(llm=llm, max_loops=5) # Define a starting task or message initial_task = "Generate an long form analysis on the transformer model architecture." @@ -126,7 +131,7 @@ from swarms.structs import Flow def stop_when_repeats(response: str) -> bool: return "Stop" in response.lower() -flow = Flow(llm=my_language_model, max_loops=5, stopping_condition=stop_when_repeats) +flow = Flow(llm=llm, max_loops=5, stopping_condition=stop_when_repeats) ``` ### Example 3: Interactive Conversation @@ -134,7 +139,7 @@ flow = Flow(llm=my_language_model, max_loops=5, stopping_condition=stop_when_rep ```python from swarms.structs import Flow -flow = Flow(llm=my_language_model, max_loops=5, interactive=True) +flow = Flow(llm=llm, max_loops=5, interactive=True) # Provide initial task initial_task = "Rank and prioritize the following financial documents and cut out 30% of our expenses" diff --git a/simple_agent.py b/playground/agents/simple_agent.py similarity index 100% rename from simple_agent.py rename to playground/agents/simple_agent.py diff --git a/playground/models/multitemp.py b/playground/models/multitemp.py deleted file mode 100644 index f4146390..00000000 --- a/playground/models/multitemp.py +++ /dev/null @@ -1,56 +0,0 @@ -from swarms.models import OpenAIChat # Replace with your actual OpenAIChat import - -if __name__ == "__main__": - api_key = "" # Your OpenAI API key here - agent = MultiTempAgent(api_key) - - prompt = "Write a blog post about health and wellness" - final_output = agent.run(prompt) - - print("Final chosen output:") - print(final_output) - - -class MultiTempAgent: - def __init__(self, api_key, default_temp=0.5, alt_temps=[0.2, 0.7, 0.9]): - self.api_key = api_key - self.default_temp = default_temp - self.alt_temps = alt_temps - - def ask_user_feedback(self, text): - print(f"Generated text: {text}") - feedback = input("Are you satisfied with this output? (yes/no): ") - return feedback.lower() == "yes" - - def present_options_to_user(self, outputs): - print("Alternative outputs:") - for temp, output in outputs.items(): - print(f"Temperature {temp}: {output}") - chosen_temp = float(input("Choose the temperature of the output you like: ")) - return outputs.get(chosen_temp, "Invalid temperature chosen.") - - def run(self, prompt): - try: - llm = OpenAIChat(openai_api_key=self.api_key, temperature=self.default_temp) - initial_output = llm(prompt) # Using llm as a callable - except Exception as e: - print(f"Error generating initial output: {e}") - initial_output = None - - user_satisfied = self.ask_user_feedback(initial_output) - - if user_satisfied: - return initial_output - else: - outputs = {} - for temp in self.alt_temps: - try: - llm = OpenAIChat( - openai_api_key=self.api_key, temperature=temp - ) # Re-initializing - outputs[temp] = llm(prompt) # Using llm as a callable - except Exception as e: - print(f"Error generating text at temperature {temp}: {e}") - outputs[temp] = None - chosen_output = self.present_options_to_user(outputs) - return chosen_output diff --git a/playground/models/openai_model.py b/playground/models/openai_model.py index eccbb8cc..e3b01715 100644 --- a/playground/models/openai_model.py +++ b/playground/models/openai_model.py @@ -1,6 +1,6 @@ from swarms.models.openai_models import OpenAIChat -openai = OpenAIChat(openai_api_key="", verbose=False) +openai = OpenAIChat(openai_api_key="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", verbose=False) -chat = openai("Are quantum fields everywhere?") +chat = openai("What are quantum fields?") print(chat) diff --git a/playground/structs/flow.py b/playground/structs/flow.py index e69de29b..8e34cce3 100644 --- a/playground/structs/flow.py +++ b/playground/structs/flow.py @@ -0,0 +1,35 @@ +from swarms.models import OpenAIChat +from swarms.structs import Flow + +api_key = "" + +# Initialize the language model, this model can be swapped out with Anthropic, ETC, Huggingface Models like Mistral, ETC +llm = OpenAIChat( + # model_name="gpt-4" + openai_api_key=api_key, + temperature=0.5, + # max_tokens=100, +) + +## Initialize the workflow +flow = Flow( + llm=llm, + max_loops=2, + dashboard=True, + # stopping_condition=None, # You can define a stopping condition as needed. + # loop_interval=1, + # retry_attempts=3, + # retry_interval=1, + # interactive=False, # Set to 'True' for interactive mode. + # dynamic_temperature=False, # Set to 'True' for dynamic temperature handling. +) + +# out = flow.load_state("flow_state.json") +# temp = flow.dynamic_temperature() +# filter = flow.add_response_filter("Trump") +out = flow.run("Generate a 10,000 word blog on health and wellness.") +# out = flow.validate_response(out) +# out = flow.analyze_feedback(out) +# out = flow.print_history_and_memory() +# # out = flow.save_state("flow_state.json") +# print(out) diff --git a/playground/structs/sequential_workflow.py b/playground/structs/sequential_workflow.py new file mode 100644 index 00000000..b8e5a10b --- /dev/null +++ b/playground/structs/sequential_workflow.py @@ -0,0 +1,31 @@ +from swarms.models import OpenAIChat +from swarms.structs import Flow +from swarms.structs.sequential_workflow import SequentialWorkflow + +# Example usage +llm = OpenAIChat( + temperature=0.5, + max_tokens=3000, +) + +# Initialize the Flow with the language flow +flow1 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create another Flow for a different task +flow2 = Flow(llm=llm, max_loops=1, dashboard=False) + +# Create the workflow +workflow = SequentialWorkflow(max_loops=1) + +# Add tasks to the workflow +workflow.add("Generate a 10,000 word blog on health and wellness.", flow1) + +# Suppose the next task takes the output of the first task as input +workflow.add("Summarize the generated blog", flow2) + +# Run the workflow +workflow.run() + +# Output the results +for task in workflow.tasks: + print(f"Task: {task.description}, Result: {task.result}") diff --git a/playground/swarms/godmode.py b/playground/swarms/godmode.py index 66aec1fa..f1269d98 100644 --- a/playground/swarms/godmode.py +++ b/playground/swarms/godmode.py @@ -1,39 +1,16 @@ +from swarms.swarms import GodMode from swarms.models import OpenAIChat -from swarms.swarms import GodMode -from swarms.workers.worker import Worker +api_key = "" + +llm = OpenAIChat(openai_api_key=api_key) -llm = OpenAIChat(model_name="gpt-4", openai_api_key="api-key", temperature=0.5) -worker1 = Worker( - llm=llm, - ai_name="Bumble Bee", - ai_role="Worker in a swarm", - external_tools=None, - human_in_the_loop=False, - temperature=0.5, -) -worker2 = Worker( - llm=llm, - ai_name="Optimus Prime", - ai_role="Worker in a swarm", - external_tools=None, - human_in_the_loop=False, - temperature=0.5, -) -worker3 = Worker( - llm=llm, - ai_name="Megatron", - ai_role="Worker in a swarm", - external_tools=None, - human_in_the_loop=False, - temperature=0.5, -) -# Usage -agents = [worker1, worker2, worker3] +llms = [llm, llm, llm] -god_mode = GodMode(agents) +god_mode = GodMode(llms) -task = "What are the biggest risks facing humanity?" +task = "Generate a 10,000 word blog on health and wellness." +out = god_mode.run(task) god_mode.print_responses(task) diff --git a/playground/swarms/groupchat.py b/playground/swarms/groupchat.py index a5e8dd0d..739181d1 100644 --- a/playground/swarms/groupchat.py +++ b/playground/swarms/groupchat.py @@ -1,61 +1,49 @@ -from swarms.models import OpenAIChat -from swarms.swarms import GroupChat, GroupChatManager -from swarms.workers import Worker +from swarms import OpenAI, Flow +from swarms.swarms.groupchat import GroupChatManager, GroupChat -llm = OpenAIChat(model_name="gpt-4", openai_api_key="api-key", temperature=0.5) -node = Worker( - llm=llm, - ai_name="Optimus Prime", - ai_role="Worker in a swarm", - external_tools=None, - human_in_the_loop=False, +api_key = "" + +llm = OpenAI( + openai_api_key=api_key, temperature=0.5, + max_tokens=3000, ) -node2 = Worker( +# Initialize the flow +flow1 = Flow( llm=llm, - ai_name="Optimus Prime", - ai_role="Worker in a swarm", - external_tools=None, - human_in_the_loop=False, - temperature=0.5, + max_loops=1, + system_message="YOU ARE SILLY, YOU OFFER NOTHING OF VALUE", + name="silly", + dashboard=True, ) - -node3 = Worker( +flow2 = Flow( llm=llm, - ai_name="Optimus Prime", - ai_role="Worker in a swarm", - external_tools=None, - human_in_the_loop=False, - temperature=0.5, + max_loops=1, + system_message="YOU ARE VERY SMART AND ANSWER RIDDLES", + name="detective", + dashboard=True, ) - -nodes = [node, node2, node3] - -messages = [ - { - "role": "system", - "context": "Create an a small feedforward in pytorch", - } -] - -group = GroupChat( - workers=nodes, - messages=messages, - max_rounds=3, +flow3 = Flow( + llm=llm, + max_loops=1, + system_message="YOU MAKE RIDDLES", + name="riddler", + dashboard=True, ) - - -manager = GroupChatManager( - groupchat=group, - max_consecutive_auto_reply=3, +manager = Flow( + llm=llm, + max_loops=1, + system_message="YOU ARE A GROUP CHAT MANAGER", + name="manager", + dashboard=True, ) -output = group.run( - messages, - sender=node, - config=group, -) -print(output) +# Example usage: +agents = [flow1, flow2, flow3] + +group_chat = GroupChat(agents=agents, messages=[], max_round=10) +chat_manager = GroupChatManager(groupchat=group_chat, selector=manager) +chat_history = chat_manager("Write me a riddle") From 97aa8bc3a02fd6c66f104d62fd6c33e1dc856228 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 17:53:02 -0500 Subject: [PATCH 28/32] saved state in dashboard error --- playground/models/openai_model.py | 2 +- pyproject.toml | 2 +- swarms/models/__init__.py | 8 ++++---- swarms/models/dalle3.py | 3 ++- swarms/models/gpt4v.py | 2 +- swarms/structs/flow.py | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/playground/models/openai_model.py b/playground/models/openai_model.py index e3b01715..3b9cb967 100644 --- a/playground/models/openai_model.py +++ b/playground/models/openai_model.py @@ -1,6 +1,6 @@ from swarms.models.openai_models import OpenAIChat -openai = OpenAIChat(openai_api_key="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", verbose=False) +openai = OpenAIChat(openai_api_key="", verbose=False) chat = openai("What are quantum fields?") print(chat) diff --git a/pyproject.toml b/pyproject.toml index d8a561bd..6aa8585d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "swarms" -version = "1.9.6" +version = "1.9.9" description = "Swarms - Pytorch" license = "MIT" authors = ["Kye Gomez "] diff --git a/swarms/models/__init__.py b/swarms/models/__init__.py index b2a2b433..dd21ba80 100644 --- a/swarms/models/__init__.py +++ b/swarms/models/__init__.py @@ -16,8 +16,8 @@ from swarms.models.kosmos_two import Kosmos from swarms.models.vilt import Vilt from swarms.models.nougat import Nougat from swarms.models.layoutlm_document_qa import LayoutLMDocumentQA -from swarms.models.gpt4v import GPT4Vision -from swarms.models.dalle3 import Dalle3 +# from swarms.models.gpt4v import GPT4Vision +# from swarms.models.dalle3 import Dalle3 # from swarms.models.distilled_whisperx import DistilWhisperModel @@ -45,6 +45,6 @@ __all__ = [ "HuggingfaceLLM", "MPT7B", "WizardLLMStoryTeller", - "GPT4Vision", - "Dalle3", + # "GPT4Vision", + # "Dalle3", ] diff --git a/swarms/models/dalle3.py b/swarms/models/dalle3.py index 73edf502..2ac5d403 100644 --- a/swarms/models/dalle3.py +++ b/swarms/models/dalle3.py @@ -12,7 +12,7 @@ from termcolor import colored load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") +# api_key = os.getenv("OPENAI_API_KEY") # Configure Logging logging.basicConfig(level=logging.INFO) @@ -49,6 +49,7 @@ class Dalle3: size: str = "1024x1024" max_retries: int = 3 quality: str = "standard" + api_key: str = None n: int = 4 client = OpenAI( api_key=api_key, diff --git a/swarms/models/gpt4v.py b/swarms/models/gpt4v.py index a7f8f1c1..99580d82 100644 --- a/swarms/models/gpt4v.py +++ b/swarms/models/gpt4v.py @@ -73,7 +73,7 @@ class GPT4Vision: model: str = "gpt-4-vision-preview" backoff_factor: float = 2.0 timeout_seconds: int = 10 - api_key: Optional[str] = None or os.getenv("OPENAI_API_KEY") + api_key: Optional[str] = None # 'Low' or 'High' for respesctively fast or high quality, but high more token usage quality: str = "low" # Max tokens to use for the API request, the maximum might be 3,000 but we don't know diff --git a/swarms/structs/flow.py b/swarms/structs/flow.py index 117172ea..4e21c3df 100644 --- a/swarms/structs/flow.py +++ b/swarms/structs/flow.py @@ -217,7 +217,7 @@ class Flow: Dashboard: {self.dashboard} Dynamic Temperature: {self.dynamic_temperature} Autosave: {self.autosave} - Saved State: {self.saved_state} + Saved State: {self.saved_state_path} ---------------------------------------- """, From 4c6bbad49b67603460ef80f4744192c2011e4e23 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 18:18:57 -0500 Subject: [PATCH 29/32] anthropic docs --- docs/prompt.txt | 93 ------------------- docs/swarms/models/anthropic.md | 11 ++- godmode.py | 16 ---- dalle3.py => playground/models/dalle3.py | 0 .../models/gpt4vision_example.py | 0 swarms/agents/idea_to_image_agent.py | 2 +- swarms/models/huggingface.py | 17 ++-- tests/models/dalle3.py | 2 +- 8 files changed, 19 insertions(+), 122 deletions(-) delete mode 100644 docs/prompt.txt delete mode 100644 godmode.py rename dalle3.py => playground/models/dalle3.py (100%) rename gpt4vision_example.py => playground/models/gpt4vision_example.py (100%) diff --git a/docs/prompt.txt b/docs/prompt.txt deleted file mode 100644 index 3644be4a..00000000 --- a/docs/prompt.txt +++ /dev/null @@ -1,93 +0,0 @@ -Create multi-page long and explicit professional pytorch-like documentation for the swarms code below follow the outline for the swarms library, provide many examples and teach the user about the code, provide examples for every function, make the documentation 10,000 words, provide many usage examples and note this is markdown docs, create the documentation for the code to document. - -Now make the professional documentation for this code, provide the architecture and how the class works and why it works that way, it's purpose, provide args, their types, 3 ways of usage examples, in examples use from shapeless import x - -BE VERY EXPLICIT AND THOROUGH, MAKE IT DEEP AND USEFUL - -######## -Step 1: Understand the purpose and functionality of the module or framework - -Read and analyze the description provided in the documentation to understand the purpose and functionality of the module or framework. -Identify the key features, parameters, and operations performed by the module or framework. -Step 2: Provide an overview and introduction - -Start the documentation by providing a brief overview and introduction to the module or framework. -Explain the importance and relevance of the module or framework in the context of the problem it solves. -Highlight any key concepts or terminology that will be used throughout the documentation. -Step 3: Provide a class or function definition - -Provide the class or function definition for the module or framework. -Include the parameters that need to be passed to the class or function and provide a brief description of each parameter. -Specify the data types and default values for each parameter. -Step 4: Explain the functionality and usage - -Provide a detailed explanation of how the module or framework works and what it does. -Describe the steps involved in using the module or framework, including any specific requirements or considerations. -Provide code examples to demonstrate the usage of the module or framework. -Explain the expected inputs and outputs for each operation or function. -Step 5: Provide additional information and tips - -Provide any additional information or tips that may be useful for using the module or framework effectively. -Address any common issues or challenges that developers may encounter and provide recommendations or workarounds. -Step 6: Include references and resources - -Include references to any external resources or research papers that provide further information or background on the module or framework. -Provide links to relevant documentation or websites for further exploration. -Example Template for the given documentation: - -# Module/Function Name: MultiheadAttention - -class torch.nn.MultiheadAttention(embed_dim, num_heads, dropout=0.0, bias=True, add_bias_kv=False, add_zero_attn=False, kdim=None, vdim=None, batch_first=False, device=None, dtype=None): - """ - Creates a multi-head attention module for joint information representation from the different subspaces. - - Parameters: - - embed_dim (int): Total dimension of the model. - - num_heads (int): Number of parallel attention heads. The embed_dim will be split across num_heads. - - dropout (float): Dropout probability on attn_output_weights. Default: 0.0 (no dropout). - - bias (bool): If specified, adds bias to input/output projection layers. Default: True. - - add_bias_kv (bool): If specified, adds bias to the key and value sequences at dim=0. Default: False. - - add_zero_attn (bool): If specified, adds a new batch of zeros to the key and value sequences at dim=1. Default: False. - - kdim (int): Total number of features for keys. Default: None (uses kdim=embed_dim). - - vdim (int): Total number of features for values. Default: None (uses vdim=embed_dim). - - batch_first (bool): If True, the input and output tensors are provided as (batch, seq, feature). Default: False. - - device (torch.device): If specified, the tensors will be moved to the specified device. - - dtype (torch.dtype): If specified, the tensors will have the specified dtype. - """ - - def forward(query, key, value, key_padding_mask=None, need_weights=True, attn_mask=None, average_attn_weights=True, is_causal=False): - """ - Forward pass of the multi-head attention module. - - Parameters: - - query (Tensor): Query embeddings of shape (L, E_q) for unbatched input, (L, N, E_q) when batch_first=False, or (N, L, E_q) when batch_first=True. - - key (Tensor): Key embeddings of shape (S, E_k) for unbatched input, (S, N, E_k) when batch_first=False, or (N, S, E_k) when batch_first=True. - - value (Tensor): Value embeddings of shape (S, E_v) for unbatched input, (S, N, E_v) when batch_first=False, or (N, S, E_v) when batch_first=True. - - key_padding_mask (Optional[Tensor]): If specified, a mask indicating elements to be ignored in key for attention computation. - - need_weights (bool): If specified, returns attention weights in addition to attention outputs. Default: True. - - attn_mask (Optional[Tensor]): If specified, a mask preventing attention to certain positions. - - average_attn_weights (bool): If true, returns averaged attention weights per head. Otherwise, returns attention weights separately per head. Note that this flag only has an effect when need_weights=True. Default: True. - - is_causal (bool): If specified, applies a causal mask as the attention mask. Default: False. - - Returns: - Tuple[Tensor, Optional[Tensor]]: - - attn_output (Tensor): Attention outputs of shape (L, E) for unbatched input, (L, N, E) when batch_first=False, or (N, L, E) when batch_first=True. - - attn_output_weights (Optional[Tensor]): Attention weights of shape (L, S) when unbatched or (N, L, S) when batched. Optional, only returned when need_weights=True. - """ - - # Implementation of the forward pass of the attention module goes here - - return attn_output, attn_output_weights - - -# Usage example: - -multihead_attn = nn.MultiheadAttention(embed_dim, num_heads) -attn_output, attn_output_weights = multihead_attn(query, key, value) -Note: - -The above template includes the class or function definition, parameters, description, and usage example. -To replicate the documentation for any other module or framework, follow the same structure and provide the specific details for that module or framework. - - -############# CODE TO DOCUMENT, DOCUMENT THE diff --git a/docs/swarms/models/anthropic.md b/docs/swarms/models/anthropic.md index 4d5f1fcd..cf139f76 100644 --- a/docs/swarms/models/anthropic.md +++ b/docs/swarms/models/anthropic.md @@ -70,17 +70,18 @@ class Anthropic: ```python # Import necessary modules and classes from swarms.models import Anthropic -import torch # Initialize an instance of the Anthropic class -anthropic_instance = Anthropic() +model = Anthropic( + anthropic_api_key="sk-" +) -# Using the generate method -completion_1 = anthropic_instance.generate("What is the capital of France?") +# Using the run method +completion_1 = model.run("What is the capital of France?") print(completion_1) # Using the __call__ method -completion_2 = anthropic_instance("How far is the moon from the earth?", stop=["miles", "km"]) +completion_2 = model("How far is the moon from the earth?", stop=["miles", "km"]) print(completion_2) ``` diff --git a/godmode.py b/godmode.py deleted file mode 100644 index f1269d98..00000000 --- a/godmode.py +++ /dev/null @@ -1,16 +0,0 @@ -from swarms.swarms import GodMode -from swarms.models import OpenAIChat - -api_key = "" - -llm = OpenAIChat(openai_api_key=api_key) - - -llms = [llm, llm, llm] - -god_mode = GodMode(llms) - -task = "Generate a 10,000 word blog on health and wellness." - -out = god_mode.run(task) -god_mode.print_responses(task) diff --git a/dalle3.py b/playground/models/dalle3.py similarity index 100% rename from dalle3.py rename to playground/models/dalle3.py diff --git a/gpt4vision_example.py b/playground/models/gpt4vision_example.py similarity index 100% rename from gpt4vision_example.py rename to playground/models/gpt4vision_example.py diff --git a/swarms/agents/idea_to_image_agent.py b/swarms/agents/idea_to_image_agent.py index e2a06691..f7e5ec0c 100644 --- a/swarms/agents/idea_to_image_agent.py +++ b/swarms/agents/idea_to_image_agent.py @@ -1,7 +1,7 @@ import os import logging from dataclasses import dataclass -from dalle3 import Dalle +from playground.models.dalle3 import Dalle from swarms.models import OpenAIChat diff --git a/swarms/models/huggingface.py b/swarms/models/huggingface.py index d18b1b9d..0c5bf2c7 100644 --- a/swarms/models/huggingface.py +++ b/swarms/models/huggingface.py @@ -23,7 +23,7 @@ class HuggingfaceLLM: ``` from swarms.models import HuggingfaceLLM - model_id = "gpt2-small" + model_id = "NousResearch/Yarn-Mistral-7b-128k" inference = HuggingfaceLLM(model_id=model_id) task = "Once upon a time" @@ -74,15 +74,20 @@ class HuggingfaceLLM: bnb_config = BitsAndBytesConfig(**quantization_config) try: - self.tokenizer = AutoTokenizer.from_pretrained(self.model_id) + self.tokenizer = AutoTokenizer.from_pretrained(self.model_id, *args, **kwargs) self.model = AutoModelForCausalLM.from_pretrained( - self.model_id, quantization_config=bnb_config + self.model_id, quantization_config=bnb_config, *args, **kwargs ) self.model # .to(self.device) except Exception as e: - self.logger.error(f"Failed to load the model or the tokenizer: {e}") - raise + # self.logger.error(f"Failed to load the model or the tokenizer: {e}") + # raise + print(colored(f"Failed to load the model and or the tokenizer: {e}", "red")) + + def print_error(self, error: str): + """Print error""" + print(colored(f"Error: {error}", "red")) def load_model(self): """Load the model""" @@ -157,7 +162,7 @@ class HuggingfaceLLM: del inputs return self.tokenizer.decode(outputs[0], skip_special_tokens=True) except Exception as e: - self.logger.error(f"Failed to generate the text: {e}") + print(colored(f"HuggingfaceLLM could not generate text because of error: {e}, try optimizing your arguments", "red")) raise async def run_async(self, task: str, *args, **kwargs) -> str: diff --git a/tests/models/dalle3.py b/tests/models/dalle3.py index ff1489ea..42b851b7 100644 --- a/tests/models/dalle3.py +++ b/tests/models/dalle3.py @@ -6,7 +6,7 @@ from openai import OpenAIError from PIL import Image from termcolor import colored -from dalle3 import Dalle3 +from playground.models.dalle3 import Dalle3 # Mocking the OpenAI client to avoid making actual API calls during testing From 8dc90648199f222582ef490a74931dbb45eb1cf2 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 19:08:13 -0500 Subject: [PATCH 30/32] clean up --- docs/swarms/models/anthropic.md | 2 +- example.py | 6 ++++-- demos/positive_med.py => positive_med.py | 0 pyproject.toml | 2 +- swarms/agents/__init__.py | 4 ++-- swarms/agents/idea_to_image_agent.py | 2 +- swarms/models/anthropic.py | 5 ++++- 7 files changed, 13 insertions(+), 8 deletions(-) rename demos/positive_med.py => positive_med.py (100%) diff --git a/docs/swarms/models/anthropic.md b/docs/swarms/models/anthropic.md index cf139f76..85e7a428 100644 --- a/docs/swarms/models/anthropic.md +++ b/docs/swarms/models/anthropic.md @@ -73,7 +73,7 @@ from swarms.models import Anthropic # Initialize an instance of the Anthropic class model = Anthropic( - anthropic_api_key="sk-" + anthropic_api_key="" ) # Using the run method diff --git a/example.py b/example.py index 8e34cce3..b3740aa2 100644 --- a/example.py +++ b/example.py @@ -11,11 +11,13 @@ llm = OpenAIChat( # max_tokens=100, ) + ## Initialize the workflow flow = Flow( llm=llm, - max_loops=2, + max_loops=5, dashboard=True, + # tools = [search_api, slack, ] # stopping_condition=None, # You can define a stopping condition as needed. # loop_interval=1, # retry_attempts=3, @@ -27,7 +29,7 @@ flow = Flow( # out = flow.load_state("flow_state.json") # temp = flow.dynamic_temperature() # filter = flow.add_response_filter("Trump") -out = flow.run("Generate a 10,000 word blog on health and wellness.") +out = flow.run("Generate a 10,000 word blog on mental clarity and the benefits of meditation.") # out = flow.validate_response(out) # out = flow.analyze_feedback(out) # out = flow.print_history_and_memory() diff --git a/demos/positive_med.py b/positive_med.py similarity index 100% rename from demos/positive_med.py rename to positive_med.py diff --git a/pyproject.toml b/pyproject.toml index 6aa8585d..a80b6389 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "swarms" -version = "1.9.9" +version = "2.0.1" description = "Swarms - Pytorch" license = "MIT" authors = ["Kye Gomez "] diff --git a/swarms/agents/__init__.py b/swarms/agents/__init__.py index 597c8c76..34dc0f1d 100644 --- a/swarms/agents/__init__.py +++ b/swarms/agents/__init__.py @@ -5,7 +5,7 @@ from swarms.agents.message import Message # from swarms.agents.stream_response import stream from swarms.agents.base import AbstractAgent from swarms.agents.registry import Registry -from swarms.agents.idea_to_image_agent import Idea2Image +# from swarms.agents.idea_to_image_agent import Idea2Image from swarms.agents.simple_agent import SimpleAgent @@ -17,6 +17,6 @@ __all__ = [ "Message", "AbstractAgent", "Registry", - "Idea2Image", + # "Idea2Image", "SimpleAgent", ] diff --git a/swarms/agents/idea_to_image_agent.py b/swarms/agents/idea_to_image_agent.py index f7e5ec0c..ce3654e0 100644 --- a/swarms/agents/idea_to_image_agent.py +++ b/swarms/agents/idea_to_image_agent.py @@ -1,7 +1,7 @@ import os import logging from dataclasses import dataclass -from playground.models.dalle3 import Dalle +from swarms.models.dalle3 import Dalle from swarms.models import OpenAIChat diff --git a/swarms/models/anthropic.py b/swarms/models/anthropic.py index e2066637..9914fce9 100644 --- a/swarms/models/anthropic.py +++ b/swarms/models/anthropic.py @@ -44,6 +44,7 @@ class Anthropic: top_p=None, streaming=False, default_request_timeout=None, + api_key: str = None ): self.model = model self.max_tokens_to_sample = max_tokens_to_sample @@ -56,6 +57,7 @@ class Anthropic: "ANTHROPIC_API_URL", "https://api.anthropic.com" ) self.anthropic_api_key = os.getenv("ANTHROPIC_API_KEY") + self.api_key = api_key def _default_params(self): """Get the default parameters for calling Anthropic API.""" @@ -73,9 +75,10 @@ class Anthropic: def run(self, task: str, stop=None): """Call out to Anthropic's completion endpoint.""" + api_key = self.api_key or self.anthropic_api_key stop = stop or [] params = self._default_params() - headers = {"Authorization": f"Bearer {self.anthropic_api_key}"} + headers = {"Authorization": f"Bearer {api_key}"} data = {"prompt": task, "stop_sequences": stop, **params} response = requests.post( f"{self.anthropic_api_url}/completions", From 62a413579cedbc8e7a0378ed6850454ee42cebe5 Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 19:18:13 -0500 Subject: [PATCH 31/32] swarms --- README.md | 48 +----------------------- positive_med.py => demos/positive_med.py | 0 2 files changed, 1 insertion(+), 47 deletions(-) rename positive_med.py => demos/positive_med.py (100%) diff --git a/README.md b/README.md index a80a307a..289a4c22 100644 --- a/README.md +++ b/README.md @@ -81,35 +81,6 @@ out = flow.run("Generate a 10,000 word blog on health and wellness.") -``` - - -## `GodMode` -- A powerful tool for concurrent execution of tasks using multiple Language Model (LLM) instances. - -```python -from swarms.swarms import GodMode -from swarms.models import OpenAIChat - -api_key = "" - -llm = OpenAIChat( - openai_api_key=api_key -) - - -llms = [ - llm, - llm, - llm -] - -god_mode = GodMode(llms) - -task = 'Generate a 10,000 word blog on health and wellness.' - -out = god_mode.run(task) -god_mode.print_responses(task) ``` ------ @@ -159,22 +130,6 @@ for task in workflow.tasks: ``` -### `OmniModalAgent` -- OmniModal Agent is an LLM that access to 10+ multi-modal encoders and diffusers! It can generate images, videos, speech, music and so much more, get started with: - -```python -from swarms.models import OpenAIChat -from swarms.agents import OmniModalAgent - -api_key = "SK-" - -llm = OpenAIChat(model_name="gpt-4", openai_api_key=api_key) - -agent = OmniModalAgent(llm) - -agent.run("Create a video of a swarm of fish") - -``` --- @@ -183,8 +138,7 @@ agent.run("Create a video of a swarm of fish") ## Contribute - -We're always looking for contributors to help us improve and expand this project. If you're interested, please check out our [Contributing Guidelines](CONTRIBUTING.md) and our [contributing board](https://github.com/users/kyegomez/projects/1) +- We're always looking for contributors to help us improve and expand this project. If you're interested, please check out our [Contributing Guidelines](CONTRIBUTING.md) and our [contributing board](https://github.com/users/kyegomez/projects/1) # License diff --git a/positive_med.py b/demos/positive_med.py similarity index 100% rename from positive_med.py rename to demos/positive_med.py From 16176e8cad15d3609ffd4567115c3f1835d69a5d Mon Sep 17 00:00:00 2001 From: Kye Date: Mon, 6 Nov 2023 21:28:20 -0500 Subject: [PATCH 32/32] removed open interpreter, clean uped docs, added add messages to flow + utils --- docs/swarms/chunkers/basechunker.md | 2 +- docs/swarms/chunkers/pdf_chunker.md | 2 +- example.py | 4 +- pyproject.toml | 4 +- requirements.txt | 1 + swarms/agents/__init__.py | 1 + swarms/agents/companion.py | 4 + swarms/agents/profitpilot.py | 8 +- swarms/chunkers/base.py | 35 +++++-- swarms/chunkers/markdown.py | 7 ++ swarms/chunkers/omni_chunker.py | 124 ++++++++++++++++++++++ swarms/chunkers/pdf.py | 7 ++ swarms/models/__init__.py | 1 + swarms/models/anthropic.py | 2 +- swarms/models/dalle3.py | 11 +- swarms/models/huggingface.py | 11 +- swarms/models/openai_assistant.py | 74 +++++++++++++ swarms/models/openai_tokenizer.py | 150 +++++++++++++++++++++++++++ swarms/structs/flow.py | 21 ++++ swarms/tools/interpreter_tool.py | 24 ----- swarms/workers/__init__.py | 2 +- tests/chunkers/basechunker.py | 4 +- tests/models/dalle3.py | 74 +++++++++---- tests/models/gpt4v.py | 155 ++++++++++++++++++++-------- 24 files changed, 608 insertions(+), 120 deletions(-) create mode 100644 swarms/agents/companion.py create mode 100644 swarms/chunkers/omni_chunker.py create mode 100644 swarms/models/openai_assistant.py create mode 100644 swarms/models/openai_tokenizer.py delete mode 100644 swarms/tools/interpreter_tool.py diff --git a/docs/swarms/chunkers/basechunker.md b/docs/swarms/chunkers/basechunker.md index fed03277..33b03312 100644 --- a/docs/swarms/chunkers/basechunker.md +++ b/docs/swarms/chunkers/basechunker.md @@ -53,7 +53,7 @@ The `BaseChunker` class is the core component of the `BaseChunker` module. It is #### Parameters: - `separators` (list[ChunkSeparator]): Specifies a list of `ChunkSeparator` objects used to split the text into chunks. -- `tokenizer` (OpenAiTokenizer): Defines the tokenizer to be used for counting tokens in the text. +- `tokenizer` (OpenAITokenizer): Defines the tokenizer to be used for counting tokens in the text. - `max_tokens` (int): Sets the maximum token limit for each chunk. ### 4.2. Examples diff --git a/docs/swarms/chunkers/pdf_chunker.md b/docs/swarms/chunkers/pdf_chunker.md index 5b97a551..8c92060d 100644 --- a/docs/swarms/chunkers/pdf_chunker.md +++ b/docs/swarms/chunkers/pdf_chunker.md @@ -52,7 +52,7 @@ The `PdfChunker` class is the core component of the `PdfChunker` module. It is u #### Parameters: - `separators` (list[ChunkSeparator]): Specifies a list of `ChunkSeparator` objects used to split the PDF text content into chunks. -- `tokenizer` (OpenAiTokenizer): Defines the tokenizer used for counting tokens in the text. +- `tokenizer` (OpenAITokenizer): Defines the tokenizer used for counting tokens in the text. - `max_tokens` (int): Sets the maximum token limit for each chunk. ### 4.2. Examples diff --git a/example.py b/example.py index b3740aa2..6c27bceb 100644 --- a/example.py +++ b/example.py @@ -29,7 +29,9 @@ flow = Flow( # out = flow.load_state("flow_state.json") # temp = flow.dynamic_temperature() # filter = flow.add_response_filter("Trump") -out = flow.run("Generate a 10,000 word blog on mental clarity and the benefits of meditation.") +out = flow.run( + "Generate a 10,000 word blog on mental clarity and the benefits of meditation." +) # out = flow.validate_response(out) # out = flow.analyze_feedback(out) # out = flow.print_history_and_memory() diff --git a/pyproject.toml b/pyproject.toml index a80b6389..3cb153c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "swarms" -version = "2.0.1" +version = "2.0.2" description = "Swarms - Pytorch" license = "MIT" authors = ["Kye Gomez "] @@ -41,6 +41,7 @@ sentencepiece = "*" wget = "*" griptape = "*" httpx = "*" +tiktoken = "*" attrs = "*" ggl = "*" beautifulsoup4 = "*" @@ -49,7 +50,6 @@ pydantic = "*" tenacity = "*" Pillow = "*" chromadb = "*" -open-interpreter = "*" tabulate = "*" termcolor = "*" black = "*" diff --git a/requirements.txt b/requirements.txt index 7ff9d362..cb0c65b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ sentencepiece duckduckgo-search agent-protocol chromadb +tiktoken open-interpreter tabulate colored diff --git a/swarms/agents/__init__.py b/swarms/agents/__init__.py index 34dc0f1d..355f0ad1 100644 --- a/swarms/agents/__init__.py +++ b/swarms/agents/__init__.py @@ -5,6 +5,7 @@ from swarms.agents.message import Message # from swarms.agents.stream_response import stream from swarms.agents.base import AbstractAgent from swarms.agents.registry import Registry + # from swarms.agents.idea_to_image_agent import Idea2Image from swarms.agents.simple_agent import SimpleAgent diff --git a/swarms/agents/companion.py b/swarms/agents/companion.py new file mode 100644 index 00000000..a630895e --- /dev/null +++ b/swarms/agents/companion.py @@ -0,0 +1,4 @@ +""" +Companion agents converse with the user about the agent the user wants to create then creates the agent with the desired attributes and traits and tools and configurations + +""" diff --git a/swarms/agents/profitpilot.py b/swarms/agents/profitpilot.py index 8f6927c4..ac1d0b44 100644 --- a/swarms/agents/profitpilot.py +++ b/swarms/agents/profitpilot.py @@ -16,7 +16,6 @@ from langchain.text_splitter import CharacterTextSplitter from langchain.vectorstores import Chroma from pydantic import BaseModel, Field from swarms.prompts.sales import SALES_AGENT_TOOLS_PROMPT, conversation_stages -from swarms.tools.interpreter_tool import compile # classes @@ -166,12 +165,7 @@ def get_tools(product_catalog): func=knowledge_base.run, description="useful for when you need to answer questions about product information", ), - # Interpreter - Tool( - name="Code Interepeter", - func=compile, - description="Useful when you need to run code locally, such as Python, Javascript, Shell, and more.", - ) + # omnimodal agent ] diff --git a/swarms/chunkers/base.py b/swarms/chunkers/base.py index 464f51e4..0fabdcef 100644 --- a/swarms/chunkers/base.py +++ b/swarms/chunkers/base.py @@ -1,10 +1,13 @@ from __future__ import annotations + from abc import ABC from typing import Optional -from attr import define, field, Factory + +from attr import Factory, define, field from griptape.artifacts import TextArtifact -from swarms.chunkers.chunk_seperators import ChunkSeparator -from griptape.tokenizers import OpenAiTokenizer + +from swarms.chunkers.chunk_seperator import ChunkSeparator +from swarms.models.openai_tokenizer import OpenAITokenizer @define @@ -16,6 +19,24 @@ class BaseChunker(ABC): Usage: -------------- + from swarms.chunkers.base import BaseChunker + from swarms.chunkers.chunk_seperator import ChunkSeparator + + class PdfChunker(BaseChunker): + DEFAULT_SEPARATORS = [ + ChunkSeparator("\n\n"), + ChunkSeparator(". "), + ChunkSeparator("! "), + ChunkSeparator("? "), + ChunkSeparator(" "), + ] + + # Example + pdf = "swarmdeck.pdf" + chunker = PdfChunker() + chunks = chunker.chunk(pdf) + print(chunks) + """ @@ -26,10 +47,10 @@ class BaseChunker(ABC): default=Factory(lambda self: self.DEFAULT_SEPARATORS, takes_self=True), kw_only=True, ) - tokenizer: OpenAiTokenizer = field( + tokenizer: OpenAITokenizer = field( default=Factory( - lambda: OpenAiTokenizer( - model=OpenAiTokenizer.DEFAULT_OPENAI_GPT_3_CHAT_MODEL + lambda: OpenAITokenizer( + model=OpenAITokenizer.DEFAULT_OPENAI_GPT_3_CHAT_MODEL ) ), kw_only=True, @@ -47,7 +68,7 @@ class BaseChunker(ABC): def _chunk_recursively( self, chunk: str, current_separator: Optional[ChunkSeparator] = None ) -> list[str]: - token_count = self.tokenizer.token_count(chunk) + token_count = self.tokenizer.count_tokens(chunk) if token_count <= self.max_tokens: return [chunk] diff --git a/swarms/chunkers/markdown.py b/swarms/chunkers/markdown.py index 6c0e755f..7836b0a7 100644 --- a/swarms/chunkers/markdown.py +++ b/swarms/chunkers/markdown.py @@ -15,3 +15,10 @@ class MarkdownChunker(BaseChunker): ChunkSeparator("? "), ChunkSeparator(" "), ] + + +# # Example using chunker to chunk a markdown file +# file = open("README.md", "r") +# text = file.read() +# chunker = MarkdownChunker() +# chunks = chunker.chunk(text) diff --git a/swarms/chunkers/omni_chunker.py b/swarms/chunkers/omni_chunker.py new file mode 100644 index 00000000..dca569ea --- /dev/null +++ b/swarms/chunkers/omni_chunker.py @@ -0,0 +1,124 @@ +""" +Omni Chunker is a chunker that chunks all files into select chunks of size x strings + +Usage: +-------------- +from swarms.chunkers.omni_chunker import OmniChunker + +# Example +pdf = "swarmdeck.pdf" +chunker = OmniChunker(chunk_size=1000, beautify=True) +chunks = chunker(pdf) +print(chunks) + + +""" +from dataclasses import dataclass +from typing import List, Optional, Callable +from termcolor import colored +import os +import sys + + + + +@dataclass +class OmniChunker: + """ + + + """ + chunk_size: int = 1000 + beautify: bool = False + use_tokenizer: bool = False + tokenizer: Optional[Callable[[str], List[str]]] = None + + + + def __call__(self, file_path: str) -> List[str]: + """ + Chunk the given file into parts of size `chunk_size`. + + Args: + file_path (str): The path to the file to chunk. + + Returns: + List[str]: A list of string chunks from the file. + """ + if not os.path.isfile(file_path): + print(colored("The file does not exist.", "red")) + return [] + + file_extension = os.path.splitext(file_path)[1] + try: + with open(file_path, "rb") as file: + content = file.read() + # Decode content based on MIME type or file extension + decoded_content = self.decode_content(content, file_extension) + chunks = self.chunk_content(decoded_content) + return chunks + + except Exception as e: + print(colored(f"Error reading file: {e}", "red")) + return [] + + def decode_content(self, content: bytes, file_extension: str) -> str: + """ + Decode the content of the file based on its MIME type or file extension. + + Args: + content (bytes): The content of the file. + file_extension (str): The file extension of the file. + + Returns: + str: The decoded content of the file. + """ + # Add logic to handle different file types based on the extension + # For simplicity, this example assumes text files encoded in utf-8 + try: + return content.decode("utf-8") + except UnicodeDecodeError as e: + print( + colored( + f"Could not decode file with extension {file_extension}: {e}", + "yellow", + ) + ) + return "" + + def chunk_content(self, content: str) -> List[str]: + """ + Split the content into chunks of size `chunk_size`. + + Args: + content (str): The content to chunk. + + Returns: + List[str]: The list of chunks. + """ + return [ + content[i : i + self.chunk_size] + for i in range(0, len(content), self.chunk_size) + ] + + def __str__(self): + return f"OmniChunker(chunk_size={self.chunk_size}, beautify={self.beautify})" + + def metrics(self): + return { + "chunk_size": self.chunk_size, + "beautify": self.beautify, + } + + def print_dashboard(self): + print( + colored( + f""" + Omni Chunker + ------------ + {self.metrics()} + """, + "cyan", + ) + ) + diff --git a/swarms/chunkers/pdf.py b/swarms/chunkers/pdf.py index 206c74f3..710134a0 100644 --- a/swarms/chunkers/pdf.py +++ b/swarms/chunkers/pdf.py @@ -10,3 +10,10 @@ class PdfChunker(BaseChunker): ChunkSeparator("? "), ChunkSeparator(" "), ] + + +# # Example +# pdf = "swarmdeck.pdf" +# chunker = PdfChunker() +# chunks = chunker.chunk(pdf) +# print(chunks) diff --git a/swarms/models/__init__.py b/swarms/models/__init__.py index dd21ba80..26c06066 100644 --- a/swarms/models/__init__.py +++ b/swarms/models/__init__.py @@ -16,6 +16,7 @@ from swarms.models.kosmos_two import Kosmos from swarms.models.vilt import Vilt from swarms.models.nougat import Nougat from swarms.models.layoutlm_document_qa import LayoutLMDocumentQA + # from swarms.models.gpt4v import GPT4Vision # from swarms.models.dalle3 import Dalle3 diff --git a/swarms/models/anthropic.py b/swarms/models/anthropic.py index 9914fce9..cc3931bb 100644 --- a/swarms/models/anthropic.py +++ b/swarms/models/anthropic.py @@ -44,7 +44,7 @@ class Anthropic: top_p=None, streaming=False, default_request_timeout=None, - api_key: str = None + api_key: str = None, ): self.model = model self.max_tokens_to_sample = max_tokens_to_sample diff --git a/swarms/models/dalle3.py b/swarms/models/dalle3.py index 2ac5d403..899564fc 100644 --- a/swarms/models/dalle3.py +++ b/swarms/models/dalle3.py @@ -129,7 +129,7 @@ class Dalle3: ) ) raise error - + def create_variations(self, img: str): """ Create variations of an image using the Dalle3 API @@ -151,14 +151,11 @@ class Dalle3: >>> img = dalle3.create_variations(img) >>> print(img) - + """ try: - response = self.client.images.create_variation( - img = open(img, "rb"), - n=self.n, - size=self.size + img=open(img, "rb"), n=self.n, size=self.size ) img = response.data[0].url @@ -172,4 +169,4 @@ class Dalle3: ) print(colored(f"Error running Dalle3: {error.http_status}", "red")) print(colored(f"Error running Dalle3: {error.error}", "red")) - raise error \ No newline at end of file + raise error diff --git a/swarms/models/huggingface.py b/swarms/models/huggingface.py index 0c5bf2c7..f11bf3df 100644 --- a/swarms/models/huggingface.py +++ b/swarms/models/huggingface.py @@ -74,7 +74,9 @@ class HuggingfaceLLM: bnb_config = BitsAndBytesConfig(**quantization_config) try: - self.tokenizer = AutoTokenizer.from_pretrained(self.model_id, *args, **kwargs) + self.tokenizer = AutoTokenizer.from_pretrained( + self.model_id, *args, **kwargs + ) self.model = AutoModelForCausalLM.from_pretrained( self.model_id, quantization_config=bnb_config, *args, **kwargs ) @@ -162,7 +164,12 @@ class HuggingfaceLLM: del inputs return self.tokenizer.decode(outputs[0], skip_special_tokens=True) except Exception as e: - print(colored(f"HuggingfaceLLM could not generate text because of error: {e}, try optimizing your arguments", "red")) + print( + colored( + f"HuggingfaceLLM could not generate text because of error: {e}, try optimizing your arguments", + "red", + ) + ) raise async def run_async(self, task: str, *args, **kwargs) -> str: diff --git a/swarms/models/openai_assistant.py b/swarms/models/openai_assistant.py new file mode 100644 index 00000000..6d0c518f --- /dev/null +++ b/swarms/models/openai_assistant.py @@ -0,0 +1,74 @@ +from typing import Dict, List, Optional +from dataclass import dataclass + +from swarms.models import OpenAI + + +@dataclass +class OpenAIAssistant: + name: str = "OpenAI Assistant" + instructions: str = None + tools: List[Dict] = None + model: str = None + openai_api_key: str = None + temperature: float = 0.5 + max_tokens: int = 100 + stop: List[str] = None + echo: bool = False + stream: bool = False + log: bool = False + presence: bool = False + dashboard: bool = False + debug: bool = False + max_loops: int = 5 + stopping_condition: Optional[str] = None + loop_interval: int = 1 + retry_attempts: int = 3 + retry_interval: int = 1 + interactive: bool = False + dynamic_temperature: bool = False + state: Dict = None + response_filters: List = None + response_filter: Dict = None + response_filter_name: str = None + response_filter_value: str = None + response_filter_type: str = None + response_filter_action: str = None + response_filter_action_value: str = None + response_filter_action_type: str = None + response_filter_action_name: str = None + client = OpenAI() + role: str = "user" + instructions: str = None + + def create_assistant(self, task: str): + assistant = self.client.create_assistant( + name=self.name, + instructions=self.instructions, + tools=self.tools, + model=self.model, + ) + return assistant + + def create_thread(self): + thread = self.client.beta.threads.create() + return thread + + def add_message_to_thread(self, thread_id: str, message: str): + message = self.client.beta.threads.add_message( + thread_id=thread_id, role=self.user, content=message + ) + return message + + def run(self, task: str): + run = self.client.beta.threads.runs.create( + thread_id=self.create_thread().id, + assistant_id=self.create_assistant().id, + instructions=self.instructions, + ) + + out = self.client.beta.threads.runs.retrieve( + thread_id=run.thread_id, run_id=run.id + ) + + return out diff --git a/swarms/models/openai_tokenizer.py b/swarms/models/openai_tokenizer.py new file mode 100644 index 00000000..b4e375cc --- /dev/null +++ b/swarms/models/openai_tokenizer.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from typing import Optional + +import tiktoken +from attr import Factory, define, field + + +@define(frozen=True) +class BaseTokenizer(ABC): + DEFAULT_STOP_SEQUENCES = ["Observation:"] + + stop_sequences: list[str] = field( + default=Factory(lambda: BaseTokenizer.DEFAULT_STOP_SEQUENCES), + kw_only=True, + ) + + @property + @abstractmethod + def max_tokens(self) -> int: + ... + + def count_tokens_left(self, text: str) -> int: + diff = self.max_tokens - self.count_tokens(text) + + if diff > 0: + return diff + else: + return 0 + + @abstractmethod + def count_tokens(self, text: str) -> int: + ... + + +@define(frozen=True) +class OpenAITokenizer(BaseTokenizer): + DEFAULT_OPENAI_GPT_3_COMPLETION_MODEL = "text-davinci-003" + DEFAULT_OPENAI_GPT_3_CHAT_MODEL = "gpt-3.5-turbo" + DEFAULT_OPENAI_GPT_4_MODEL = "gpt-4" + DEFAULT_ENCODING = "cl100k_base" + DEFAULT_MAX_TOKENS = 2049 + TOKEN_OFFSET = 8 + + MODEL_PREFIXES_TO_MAX_TOKENS = { + "gpt-4-32k": 32768, + "gpt-4": 8192, + "gpt-3.5-turbo-16k": 16384, + "gpt-3.5-turbo": 4096, + "gpt-35-turbo-16k": 16384, + "gpt-35-turbo": 4096, + "text-davinci-003": 4097, + "text-davinci-002": 4097, + "code-davinci-002": 8001, + "text-embedding-ada-002": 8191, + "text-embedding-ada-001": 2046, + } + + EMBEDDING_MODELS = ["text-embedding-ada-002", "text-embedding-ada-001"] + + model: str = field(kw_only=True) + + @property + def encoding(self) -> tiktoken.Encoding: + try: + return tiktoken.encoding_for_model(self.model) + except KeyError: + return tiktoken.get_encoding(self.DEFAULT_ENCODING) + + @property + def max_tokens(self) -> int: + tokens = next( + v + for k, v in self.MODEL_PREFIXES_TO_MAX_TOKENS.items() + if self.model.startswith(k) + ) + offset = 0 if self.model in self.EMBEDDING_MODELS else self.TOKEN_OFFSET + + return (tokens if tokens else self.DEFAULT_MAX_TOKENS) - offset + + def count_tokens( + self, text: str | list, model: Optional[str] = None + ) -> int: + """ + Handles the special case of ChatML. Implementation adopted from the official OpenAI notebook: + https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb + """ + if isinstance(text, list): + model = model if model else self.model + + try: + encoding = tiktoken.encoding_for_model(model) + except KeyError: + logging.warning("model not found. Using cl100k_base encoding.") + + encoding = tiktoken.get_encoding("cl100k_base") + + if model in { + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-16k-0613", + "gpt-4-0314", + "gpt-4-32k-0314", + "gpt-4-0613", + "gpt-4-32k-0613", + }: + tokens_per_message = 3 + tokens_per_name = 1 + elif model == "gpt-3.5-turbo-0301": + # every message follows <|start|>{role/name}\n{content}<|end|>\n + tokens_per_message = 4 + # if there's a name, the role is omitted + tokens_per_name = -1 + elif "gpt-3.5-turbo" in model or "gpt-35-turbo" in model: + logging.info( + "gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613." + ) + return self.count_tokens(text, model="gpt-3.5-turbo-0613") + elif "gpt-4" in model: + logging.info( + "gpt-4 may update over time. Returning num tokens assuming gpt-4-0613." + ) + return self.count_tokens(text, model="gpt-4-0613") + else: + raise NotImplementedError( + f"""token_count() is not implemented for model {model}. + See https://github.com/openai/openai-python/blob/main/chatml.md for + information on how messages are converted to tokens.""" + ) + + num_tokens = 0 + + for message in text: + num_tokens += tokens_per_message + for key, value in message.items(): + num_tokens += len(encoding.encode(value)) + if key == "name": + num_tokens += tokens_per_name + + # every reply is primed with <|start|>assistant<|message|> + num_tokens += 3 + + return num_tokens + else: + return len( + self.encoding.encode( + text, allowed_special=set(self.stop_sequences) + ) + ) \ No newline at end of file diff --git a/swarms/structs/flow.py b/swarms/structs/flow.py index 4e21c3df..9ff021f4 100644 --- a/swarms/structs/flow.py +++ b/swarms/structs/flow.py @@ -116,6 +116,7 @@ class Flow: dynamic_temperature: bool = False, saved_state_path: Optional[str] = "flow_state.json", autosave: bool = False, + context_length: int = 8192, **kwargs: Any, ): self.llm = llm @@ -188,6 +189,26 @@ class Flow: return "\n".join(params_str_list) + def truncate_history(self): + """ + Take the history and truncate it to fit into the model context length + """ + truncated_history = self.memory[-1][-self.context_length :] + self.memory[-1] = truncated_history + + def add_task_to_memory(self, task: str): + """Add the task to the memory""" + self.memory.append([f"Human: {task}"]) + + def add_message_to_memory(self, message: str): + """Add the message to the memory""" + self.memory[-1].append(message) + + def add_message_to_memory_and_truncate(self, message: str): + """Add the message to the memory and truncate""" + self.memory[-1].append(message) + self.truncate_history() + def print_dashboard(self, task: str): """Print dashboard""" model_config = self.get_llm_init_params() diff --git a/swarms/tools/interpreter_tool.py b/swarms/tools/interpreter_tool.py deleted file mode 100644 index 22758de6..00000000 --- a/swarms/tools/interpreter_tool.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import interpreter - - -def compile(task: str): - """ - Open Interpreter lets LLMs run code (Python, Javascript, Shell, and more) locally. You can chat with Open Interpreter through a ChatGPT-like interface in your terminal by running $ interpreter after installing. - - This provides a natural-language interface to your computer's general-purpose capabilities: - - Create and edit photos, videos, PDFs, etc. - Control a Chrome browser to perform research - Plot, clean, and analyze large datasets - ...etc. - ⚠️ Note: You'll be asked to approve code before it's run. - """ - - task = interpreter.chat(task, return_messages=True) - interpreter.chat() - interpreter.reset(task) - - os.environ["INTERPRETER_CLI_AUTO_RUN"] = True - os.environ["INTERPRETER_CLI_FAST_MODE"] = True - os.environ["INTERPRETER_CLI_DEBUG"] = True diff --git a/swarms/workers/__init__.py b/swarms/workers/__init__.py index 2a7cc4f1..9dabe94d 100644 --- a/swarms/workers/__init__.py +++ b/swarms/workers/__init__.py @@ -1,2 +1,2 @@ -from swarms.workers.worker import Worker +# from swarms.workers.worker import Worker from swarms.workers.base import AbstractWorker diff --git a/tests/chunkers/basechunker.py b/tests/chunkers/basechunker.py index f70705bc..4fd92da1 100644 --- a/tests/chunkers/basechunker.py +++ b/tests/chunkers/basechunker.py @@ -3,7 +3,7 @@ from swarms.chunkers.base import ( BaseChunker, TextArtifact, ChunkSeparator, - OpenAiTokenizer, + OpenAITokenizer, ) # adjust the import paths accordingly @@ -21,7 +21,7 @@ def test_default_separators(): def test_default_tokenizer(): chunker = BaseChunker() - assert isinstance(chunker.tokenizer, OpenAiTokenizer) + assert isinstance(chunker.tokenizer, OpenAITokenizer) # 2. Test Basic Chunking diff --git a/tests/models/dalle3.py b/tests/models/dalle3.py index 42b851b7..f9a2f8cf 100644 --- a/tests/models/dalle3.py +++ b/tests/models/dalle3.py @@ -23,8 +23,12 @@ def dalle3(mock_openai_client): def test_dalle3_call_success(dalle3, mock_openai_client): # Arrange task = "A painting of a dog" - expected_img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" - mock_openai_client.images.generate.return_value = Mock(data=[Mock(url=expected_img_url)]) + expected_img_url = ( + "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + ) + mock_openai_client.images.generate.return_value = Mock( + data=[Mock(url=expected_img_url)] + ) # Act img_url = dalle3(task) @@ -40,7 +44,9 @@ def test_dalle3_call_failure(dalle3, mock_openai_client, capsys): expected_error_message = "Error running Dalle3: API Error" # Mocking OpenAIError - mock_openai_client.images.generate.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + mock_openai_client.images.generate.side_effect = OpenAIError( + expected_error_message, http_status=500, error="Internal Server Error" + ) # Act and assert with pytest.raises(OpenAIError) as excinfo: @@ -57,8 +63,12 @@ def test_dalle3_call_failure(dalle3, mock_openai_client, capsys): def test_dalle3_create_variations_success(dalle3, mock_openai_client): # Arrange img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" - expected_variation_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" - mock_openai_client.images.create_variation.return_value = Mock(data=[Mock(url=expected_variation_url)]) + expected_variation_url = ( + "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" + ) + mock_openai_client.images.create_variation.return_value = Mock( + data=[Mock(url=expected_variation_url)] + ) # Act variation_img_url = dalle3.create_variations(img_url) @@ -78,7 +88,9 @@ def test_dalle3_create_variations_failure(dalle3, mock_openai_client, capsys): expected_error_message = "Error running Dalle3: API Error" # Mocking OpenAIError - mock_openai_client.images.create_variation.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + mock_openai_client.images.create_variation.side_effect = OpenAIError( + expected_error_message, http_status=500, error="Internal Server Error" + ) # Act and assert with pytest.raises(OpenAIError) as excinfo: @@ -86,7 +98,7 @@ def test_dalle3_create_variations_failure(dalle3, mock_openai_client, capsys): assert str(excinfo.value) == expected_error_message mock_openai_client.images.create_variation.assert_called_once() - + # Ensure the error message is printed in red captured = capsys.readouterr() assert colored(expected_error_message, "red") in captured.out @@ -142,8 +154,12 @@ def test_dalle3_convert_to_bytesio(): def test_dalle3_call_multiple_times(dalle3, mock_openai_client): # Arrange task = "A painting of a dog" - expected_img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" - mock_openai_client.images.generate.return_value = Mock(data=[Mock(url=expected_img_url)]) + expected_img_url = ( + "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + ) + mock_openai_client.images.generate.return_value = Mock( + data=[Mock(url=expected_img_url)] + ) # Act img_url1 = dalle3(task) @@ -159,7 +175,9 @@ def test_dalle3_call_with_large_input(dalle3, mock_openai_client): # Arrange task = "A" * 2048 # Input longer than API's limit expected_error_message = "Error running Dalle3: API Error" - mock_openai_client.images.generate.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + mock_openai_client.images.generate.side_effect = OpenAIError( + expected_error_message, http_status=500, error="Internal Server Error" + ) # Act and assert with pytest.raises(OpenAIError) as excinfo: @@ -204,7 +222,9 @@ def test_dalle3_convert_to_bytesio_invalid_format(dalle3): def test_dalle3_call_with_retry(dalle3, mock_openai_client): # Arrange task = "A painting of a dog" - expected_img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + expected_img_url = ( + "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + ) # Simulate a retry scenario mock_openai_client.images.generate.side_effect = [ @@ -223,7 +243,9 @@ def test_dalle3_call_with_retry(dalle3, mock_openai_client): def test_dalle3_create_variations_with_retry(dalle3, mock_openai_client): # Arrange img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" - expected_variation_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" + expected_variation_url = ( + "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" + ) # Simulate a retry scenario mock_openai_client.images.create_variation.side_effect = [ @@ -245,7 +267,9 @@ def test_dalle3_call_exception_logging(dalle3, mock_openai_client, capsys): expected_error_message = "Error running Dalle3: API Error" # Mocking OpenAIError - mock_openai_client.images.generate.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + mock_openai_client.images.generate.side_effect = OpenAIError( + expected_error_message, http_status=500, error="Internal Server Error" + ) # Act with pytest.raises(OpenAIError): @@ -262,7 +286,9 @@ def test_dalle3_create_variations_exception_logging(dalle3, mock_openai_client, expected_error_message = "Error running Dalle3: API Error" # Mocking OpenAIError - mock_openai_client.images.create_variation.side_effect = OpenAIError(expected_error_message, http_status=500, error="Internal Server Error") + mock_openai_client.images.create_variation.side_effect = OpenAIError( + expected_error_message, http_status=500, error="Internal Server Error" + ) # Act with pytest.raises(OpenAIError): @@ -313,7 +339,9 @@ def test_dalle3_call_with_retry_max_retries_exceeded(dalle3, mock_openai_client) task = "A painting of a dog" # Simulate max retries exceeded - mock_openai_client.images.generate.side_effect = OpenAIError("Temporary error", http_status=500, error="Internal Server Error") + mock_openai_client.images.generate.side_effect = OpenAIError( + "Temporary error", http_status=500, error="Internal Server Error" + ) # Act and assert with pytest.raises(OpenAIError) as excinfo: @@ -322,12 +350,16 @@ def test_dalle3_call_with_retry_max_retries_exceeded(dalle3, mock_openai_client) assert "Retry limit exceeded" in str(excinfo.value) -def test_dalle3_create_variations_with_retry_max_retries_exceeded(dalle3, mock_openai_client): +def test_dalle3_create_variations_with_retry_max_retries_exceeded( + dalle3, mock_openai_client +): # Arrange img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" # Simulate max retries exceeded - mock_openai_client.images.create_variation.side_effect = OpenAIError("Temporary error", http_status=500, error="Internal Server Error") + mock_openai_client.images.create_variation.side_effect = OpenAIError( + "Temporary error", http_status=500, error="Internal Server Error" + ) # Act and assert with pytest.raises(OpenAIError) as excinfo: @@ -339,7 +371,9 @@ def test_dalle3_create_variations_with_retry_max_retries_exceeded(dalle3, mock_o def test_dalle3_call_retry_with_success(dalle3, mock_openai_client): # Arrange task = "A painting of a dog" - expected_img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + expected_img_url = ( + "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" + ) # Simulate success after a retry mock_openai_client.images.generate.side_effect = [ @@ -358,7 +392,9 @@ def test_dalle3_call_retry_with_success(dalle3, mock_openai_client): def test_dalle3_create_variations_retry_with_success(dalle3, mock_openai_client): # Arrange img_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_01J9J5ZKJZJY9.png" - expected_variation_url = "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" + expected_variation_url = ( + "https://cdn.openai.com/dall-e/encoded/feats/feats_02ABCDE.png" + ) # Simulate success after a retry mock_openai_client.images.create_variation.side_effect = [ diff --git a/tests/models/gpt4v.py b/tests/models/gpt4v.py index 40ccc7f5..23e97d03 100644 --- a/tests/models/gpt4v.py +++ b/tests/models/gpt4v.py @@ -12,19 +12,22 @@ load_dotenv api_key = os.getenv("OPENAI_API_KEY") + # Mock the OpenAI client @pytest.fixture def mock_openai_client(): return Mock() + @pytest.fixture def gpt4vision(mock_openai_client): return GPT4Vision(client=mock_openai_client) + def test_gpt4vision_default_values(): # Arrange and Act gpt4vision = GPT4Vision() - + # Assert assert gpt4vision.max_retries == 3 assert gpt4vision.model == "gpt-4-vision-preview" @@ -34,59 +37,68 @@ def test_gpt4vision_default_values(): assert gpt4vision.quality == "low" assert gpt4vision.max_tokens == 200 + def test_gpt4vision_api_key_from_env_variable(): # Arrange - api_key = os.environ["OPENAI_API_KEY"] - + api_key = os.environ["OPENAI_API_KEY"] + # Act gpt4vision = GPT4Vision() - + # Assert assert gpt4vision.api_key == api_key + def test_gpt4vision_set_api_key(): # Arrange gpt4vision = GPT4Vision(api_key=api_key) - + # Assert assert gpt4vision.api_key == api_key + def test_gpt4vision_invalid_max_retries(): # Arrange and Act with pytest.raises(ValueError): GPT4Vision(max_retries=-1) + def test_gpt4vision_invalid_backoff_factor(): # Arrange and Act with pytest.raises(ValueError): GPT4Vision(backoff_factor=-1) + def test_gpt4vision_invalid_timeout_seconds(): # Arrange and Act with pytest.raises(ValueError): GPT4Vision(timeout_seconds=-1) + def test_gpt4vision_invalid_max_tokens(): # Arrange and Act with pytest.raises(ValueError): GPT4Vision(max_tokens=-1) + def test_gpt4vision_logger_initialized(): # Arrange gpt4vision = GPT4Vision() - + # Assert assert isinstance(gpt4vision.logger, logging.Logger) + def test_gpt4vision_process_img_nonexistent_file(): # Arrange gpt4vision = GPT4Vision() img_path = "nonexistent_image.jpg" - + # Act and Assert with pytest.raises(FileNotFoundError): gpt4vision.process_img(img_path) + def test_gpt4vision_call_single_task_single_image_no_openai_client(gpt4vision): # Arrange img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" @@ -96,7 +108,10 @@ def test_gpt4vision_call_single_task_single_image_no_openai_client(gpt4vision): with pytest.raises(AttributeError): gpt4vision(img_url, [task]) -def test_gpt4vision_call_single_task_single_image_empty_response(gpt4vision, mock_openai_client): + +def test_gpt4vision_call_single_task_single_image_empty_response( + gpt4vision, mock_openai_client +): # Arrange img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" task = "Describe this image." @@ -110,7 +125,10 @@ def test_gpt4vision_call_single_task_single_image_empty_response(gpt4vision, moc assert response.answer == "" mock_openai_client.chat.completions.create.assert_called_once() -def test_gpt4vision_call_multiple_tasks_single_image_empty_responses(gpt4vision, mock_openai_client): + +def test_gpt4vision_call_multiple_tasks_single_image_empty_responses( + gpt4vision, mock_openai_client +): # Arrange img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" tasks = ["Describe this image.", "What's in this picture?"] @@ -122,20 +140,30 @@ def test_gpt4vision_call_multiple_tasks_single_image_empty_responses(gpt4vision, # Assert assert all(response.answer == "" for response in responses) - assert mock_openai_client.chat.completions.create.call_count == 1 # Should be called only once + assert ( + mock_openai_client.chat.completions.create.call_count == 1 + ) # Should be called only once + -def test_gpt4vision_call_single_task_single_image_timeout(gpt4vision, mock_openai_client): +def test_gpt4vision_call_single_task_single_image_timeout( + gpt4vision, mock_openai_client +): # Arrange img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" task = "Describe this image." - mock_openai_client.chat.completions.create.side_effect = Timeout("Request timed out") + mock_openai_client.chat.completions.create.side_effect = Timeout( + "Request timed out" + ) # Act and Assert with pytest.raises(Timeout): gpt4vision(img_url, [task]) -def test_gpt4vision_call_retry_with_success_after_timeout(gpt4vision, mock_openai_client): + +def test_gpt4vision_call_retry_with_success_after_timeout( + gpt4vision, mock_openai_client +): # Arrange img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" task = "Describe this image." @@ -143,7 +171,11 @@ def test_gpt4vision_call_retry_with_success_after_timeout(gpt4vision, mock_opena # Simulate success after a timeout and retry mock_openai_client.chat.completions.create.side_effect = [ Timeout("Request timed out"), - {"choices": [{"message": {"content": {"text": "A description of the image."}}}],} + { + "choices": [ + {"message": {"content": {"text": "A description of the image."}}} + ], + }, ] # Act @@ -151,7 +183,9 @@ def test_gpt4vision_call_retry_with_success_after_timeout(gpt4vision, mock_opena # Assert assert response.answer == "A description of the image." - assert mock_openai_client.chat.completions.create.call_count == 2 # Should be called twice + assert ( + mock_openai_client.chat.completions.create.call_count == 2 + ) # Should be called twice def test_gpt4vision_process_img(): @@ -173,7 +207,9 @@ def test_gpt4vision_call_single_task_single_image(gpt4vision, mock_openai_client expected_response = GPT4VisionResponse(answer="A description of the image.") - mock_openai_client.chat.completions.create.return_value.choices[0].text = expected_response.answer + mock_openai_client.chat.completions.create.return_value.choices[ + 0 + ].text = expected_response.answer # Act response = gpt4vision(img_url, [task]) @@ -190,7 +226,9 @@ def test_gpt4vision_call_single_task_multiple_images(gpt4vision, mock_openai_cli expected_response = GPT4VisionResponse(answer="Descriptions of the images.") - mock_openai_client.chat.completions.create.return_value.choices[0].text = expected_response.answer + mock_openai_client.chat.completions.create.return_value.choices[ + 0 + ].text = expected_response.answer # Act response = gpt4vision(img_urls, [task]) @@ -213,57 +251,76 @@ def test_gpt4vision_call_multiple_tasks_single_image(gpt4vision, mock_openai_cli def create_mock_response(response): return {"choices": [{"message": {"content": {"text": response.answer}}}]} - mock_openai_client.chat.completions.create.side_effect = [create_mock_response(response) for response in expected_responses] + mock_openai_client.chat.completions.create.side_effect = [ + create_mock_response(response) for response in expected_responses + ] # Act responses = gpt4vision(img_url, tasks) # Assert assert responses == expected_responses - assert mock_openai_client.chat.completions.create.call_count == 1 # Should be called only once - def test_gpt4vision_call_multiple_tasks_single_image(gpt4vision, mock_openai_client): - # Arrange - img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" - tasks = ["Describe this image.", "What's in this picture?"] - - expected_responses = [ - GPT4VisionResponse(answer="A description of the image."), - GPT4VisionResponse(answer="It contains various objects."), + assert ( + mock_openai_client.chat.completions.create.call_count == 1 + ) # Should be called only once + + def test_gpt4vision_call_multiple_tasks_single_image( + gpt4vision, mock_openai_client + ): + # Arrange + img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" + tasks = ["Describe this image.", "What's in this picture?"] + + expected_responses = [ + GPT4VisionResponse(answer="A description of the image."), + GPT4VisionResponse(answer="It contains various objects."), + ] + + mock_openai_client.chat.completions.create.side_effect = [ + { + "choices": [ + {"message": {"content": {"text": expected_responses[i].answer}}} ] + } + for i in range(len(expected_responses)) + ] - mock_openai_client.chat.completions.create.side_effect = [ - {"choices": [{"message": {"content": {"text": expected_responses[i].answer}}}] } for i in range(len(expected_responses)) - ] + # Act + responses = gpt4vision(img_url, tasks) - # Act - responses = gpt4vision(img_url, tasks) - - # Assert - assert responses == expected_responses - assert mock_openai_client.chat.completions.create.call_count == 1 # Should be called only once + # Assert + assert responses == expected_responses + assert ( + mock_openai_client.chat.completions.create.call_count == 1 + ) # Should be called only once def test_gpt4vision_call_multiple_tasks_multiple_images(gpt4vision, mock_openai_client): # Arrange - img_urls = ["https://images.unsplash.com/photo-1694734479857-626882b6db37?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D", "https://images.unsplash.com/photo-1694734479898-6ac4633158ac?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D"] + img_urls = [ + "https://images.unsplash.com/photo-1694734479857-626882b6db37?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D", + "https://images.unsplash.com/photo-1694734479898-6ac4633158ac?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D", + ] tasks = ["Describe these images.", "What's in these pictures?"] expected_responses = [ GPT4VisionResponse(answer="Descriptions of the images."), - GPT4VisionResponse(answer="They contain various objects.") + GPT4VisionResponse(answer="They contain various objects."), ] mock_openai_client.chat.completions.create.side_effect = [ - {"choices": [{"message": {"content": {"text": response.answer}}}] } for response in expected_responses + {"choices": [{"message": {"content": {"text": response.answer}}}]} + for response in expected_responses ] # Act responses = gpt4vision(img_urls, tasks) - # Assert assert responses == expected_responses - assert mock_openai_client.chat.completions.create.call_count == 1 # Should be called only once + assert ( + mock_openai_client.chat.completions.create.call_count == 1 + ) # Should be called only once def test_gpt4vision_call_http_error(gpt4vision, mock_openai_client): @@ -283,7 +340,9 @@ def test_gpt4vision_call_request_error(gpt4vision, mock_openai_client): img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" task = "Describe this image." - mock_openai_client.chat.completions.create.side_effect = RequestException("Request Error") + mock_openai_client.chat.completions.create.side_effect = RequestException( + "Request Error" + ) # Act and Assert with pytest.raises(RequestException): @@ -295,7 +354,9 @@ def test_gpt4vision_call_connection_error(gpt4vision, mock_openai_client): img_url = "https://images.unsplash.com/photo-1694734479942-8cc7f4660578?q=80&w=1287&auto=format&fit=crop&ixlib=rb-4.0.3&ixid=M3wxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8fA%3D%3D" task = "Describe this image." - mock_openai_client.chat.completions.create.side_effect = ConnectionError("Connection Error") + mock_openai_client.chat.completions.create.side_effect = ConnectionError( + "Connection Error" + ) # Act and Assert with pytest.raises(ConnectionError): @@ -310,7 +371,9 @@ def test_gpt4vision_call_retry_with_success(gpt4vision, mock_openai_client): # Simulate success after a retry mock_openai_client.chat.completions.create.side_effect = [ RequestException("Temporary error"), - {"choices": [{"text": "A description of the image."}]} # fixed dictionary syntax + { + "choices": [{"text": "A description of the image."}] + }, # fixed dictionary syntax ] # Act @@ -318,4 +381,6 @@ def test_gpt4vision_call_retry_with_success(gpt4vision, mock_openai_client): # Assert assert response.answer == "A description of the image." - assert mock_openai_client.chat.completions.create.call_count == 2 # Should be called twice + assert ( + mock_openai_client.chat.completions.create.call_count == 2 + ) # Should be called twice