From c9f69d5fef78f132d8f4c059f4146ae77c52a57d Mon Sep 17 00:00:00 2001 From: Patrick Devaney Date: Tue, 24 Dec 2024 20:21:12 -0500 Subject: [PATCH] multi platform spreadsheet_swarm, deps build clusterops from github source --- pyproject.toml | 2 +- swarms/structs/concurrent_workflow.py | 12 +++-- tests/structs/test_spreadsheet_swarm.py | 17 +----- tests/structs/test_spreadsheet_swarm_2.py | 65 +++++++++++++++++++++++ 4 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 tests/structs/test_spreadsheet_swarm_2.py diff --git a/pyproject.toml b/pyproject.toml index a2516e75..f486ab4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,6 @@ docstring_parser = "0.16" # TODO: tiktoken = "*" networkx = "*" aiofiles = "*" -clusterops = "*" # chromadb = "*" reportlab = "*" doc-master = "*" @@ -81,6 +80,7 @@ rich = "*" # sentence-transformers = "*" swarm-models = "*" termcolor = "*" +clusterops = { git = "https://github.com/patrickbdevaney/clusterops.git", branch = "main" } # [tool.poetry.extras] diff --git a/swarms/structs/concurrent_workflow.py b/swarms/structs/concurrent_workflow.py index 9df994c3..9b671448 100644 --- a/swarms/structs/concurrent_workflow.py +++ b/swarms/structs/concurrent_workflow.py @@ -4,26 +4,30 @@ import uuid from concurrent.futures import ThreadPoolExecutor from datetime import datetime from typing import Any, Dict, List, Optional, Union +import concurrent from pydantic import BaseModel, Field from tenacity import retry, stop_after_attempt, wait_exponential -from swarms.structs.agent import Agent -from swarms.structs.base_swarm import BaseSwarm -from swarms.utils.file_processing import create_file_in_folder -import concurrent from clusterops import ( execute_on_gpu, execute_with_cpu_cores, execute_on_multiple_gpus, list_available_gpus, ) + +from swarms.structs.agent import Agent +from swarms.structs.base_swarm import BaseSwarm +from swarms.utils.file_processing import create_file_in_folder + from swarms.utils.loguru_logger import initialize_logger from swarms.structs.swarm_id_generator import generate_swarm_id logger = initialize_logger(log_folder="concurrent_workflow") + + class AgentOutputSchema(BaseModel): run_id: Optional[str] = Field( ..., description="Unique ID for the run" diff --git a/tests/structs/test_spreadsheet_swarm.py b/tests/structs/test_spreadsheet_swarm.py index 6f8c544f..4f1346c1 100644 --- a/tests/structs/test_spreadsheet_swarm.py +++ b/tests/structs/test_spreadsheet_swarm.py @@ -1,17 +1,14 @@ import os from datetime import datetime from uuid import uuid4 - # Import necessary classes from your swarm module from swarms.structs.agent import Agent from swarms.structs.base_swarm import BaseSwarm from swarms.telemetry.capture_sys_data import log_agent_data from swarms.utils.file_processing import create_file_in_folder from swarms import SpreadSheetSwarm - # Ensure you have an environment variable or default workspace dir workspace_dir = os.getenv("WORKSPACE_DIR", "./workspace") - def create_agents(num_agents: int): """ Create a list of agent instances. @@ -27,14 +24,11 @@ def create_agents(num_agents: int): agent_name = f"Agent-{i + 1}" agents.append(Agent(agent_name=agent_name)) return agents - def main(): # Number of agents to create num_agents = 5 - # Create the agents agents = create_agents(num_agents) - # Initialize the swarm with agents and other configurations swarm = SpreadSheetSwarm( name="Test-Swarm", @@ -44,35 +38,28 @@ def main(): max_loops=2, workspace_dir=workspace_dir ) - # Run a sample task in the swarm (synchronously) task = "process_data" # Ensure the run method is synchronous swarm_metadata = swarm.run(task) # Assuming this is made synchronous - # Print swarm metadata after task completion print("Swarm Metadata:") print(swarm_metadata) - # Check if CSV file has been created and saved if os.path.exists(swarm.save_file_path): print(f"Metadata saved to: {swarm.save_file_path}") else: print(f"Metadata not saved correctly. Check the save path.") - # Test saving metadata to JSON file swarm.data_to_json_file() - # Test exporting metadata to JSON swarm_json = swarm.export_to_json() print("Exported JSON metadata:") print(swarm_json) - - # Log agent data (without ClusterOps imports) + # Log agent data print("Logging agent data:") print(log_agent_data(swarm.metadata.model_dump())) - # Run the synchronous main function if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/tests/structs/test_spreadsheet_swarm_2.py b/tests/structs/test_spreadsheet_swarm_2.py new file mode 100644 index 00000000..4f1346c1 --- /dev/null +++ b/tests/structs/test_spreadsheet_swarm_2.py @@ -0,0 +1,65 @@ +import os +from datetime import datetime +from uuid import uuid4 +# Import necessary classes from your swarm module +from swarms.structs.agent import Agent +from swarms.structs.base_swarm import BaseSwarm +from swarms.telemetry.capture_sys_data import log_agent_data +from swarms.utils.file_processing import create_file_in_folder +from swarms import SpreadSheetSwarm +# Ensure you have an environment variable or default workspace dir +workspace_dir = os.getenv("WORKSPACE_DIR", "./workspace") +def create_agents(num_agents: int): + """ + Create a list of agent instances. + + Args: + num_agents (int): The number of agents to create. + + Returns: + List[Agent]: List of created Agent objects. + """ + agents = [] + for i in range(num_agents): + agent_name = f"Agent-{i + 1}" + agents.append(Agent(agent_name=agent_name)) + return agents +def main(): + # Number of agents to create + num_agents = 5 + # Create the agents + agents = create_agents(num_agents) + # Initialize the swarm with agents and other configurations + swarm = SpreadSheetSwarm( + name="Test-Swarm", + description="A swarm for testing purposes.", + agents=agents, + autosave_on=True, + max_loops=2, + workspace_dir=workspace_dir + ) + # Run a sample task in the swarm (synchronously) + task = "process_data" + + # Ensure the run method is synchronous + swarm_metadata = swarm.run(task) # Assuming this is made synchronous + # Print swarm metadata after task completion + print("Swarm Metadata:") + print(swarm_metadata) + # Check if CSV file has been created and saved + if os.path.exists(swarm.save_file_path): + print(f"Metadata saved to: {swarm.save_file_path}") + else: + print(f"Metadata not saved correctly. Check the save path.") + # Test saving metadata to JSON file + swarm.data_to_json_file() + # Test exporting metadata to JSON + swarm_json = swarm.export_to_json() + print("Exported JSON metadata:") + print(swarm_json) + # Log agent data + print("Logging agent data:") + print(log_agent_data(swarm.metadata.model_dump())) +# Run the synchronous main function +if __name__ == "__main__": + main() \ No newline at end of file