Kye Gomez
5 days ago
parent
c6df819644
commit
610e02fe8c
|
Can't render this file because it is too large.
|
@ -0,0 +1,81 @@
|
|||||||
|
from swarms import Agent, SwarmRouter
|
||||||
|
|
||||||
|
# Agent 1: Risk Metrics Calculator
|
||||||
|
risk_metrics_agent = Agent(
|
||||||
|
agent_name="Risk-Metrics-Calculator",
|
||||||
|
agent_description="Calculates key risk metrics like VaR, Sharpe ratio, and volatility",
|
||||||
|
system_prompt="""You are a risk metrics specialist. Calculate and explain:
|
||||||
|
- Value at Risk (VaR)
|
||||||
|
- Sharpe ratio
|
||||||
|
- Volatility
|
||||||
|
- Maximum drawdown
|
||||||
|
- Beta coefficient
|
||||||
|
|
||||||
|
Provide clear, numerical results with brief explanations.""",
|
||||||
|
max_loops=1,
|
||||||
|
model_name="gpt-4.1",
|
||||||
|
random_model_enabled=True,
|
||||||
|
dynamic_temperature_enabled=True,
|
||||||
|
output_type="str-all-except-first",
|
||||||
|
max_tokens=4096,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Agent 2: Portfolio Risk Analyzer
|
||||||
|
portfolio_risk_agent = Agent(
|
||||||
|
agent_name="Portfolio-Risk-Analyzer",
|
||||||
|
agent_description="Analyzes portfolio diversification and concentration risk",
|
||||||
|
system_prompt="""You are a portfolio risk analyst. Focus on:
|
||||||
|
- Portfolio diversification analysis
|
||||||
|
- Concentration risk assessment
|
||||||
|
- Correlation analysis
|
||||||
|
- Sector/asset allocation risk
|
||||||
|
- Liquidity risk evaluation
|
||||||
|
|
||||||
|
Provide actionable insights for risk reduction.""",
|
||||||
|
max_loops=1,
|
||||||
|
model_name="gpt-4.1",
|
||||||
|
random_model_enabled=True,
|
||||||
|
dynamic_temperature_enabled=True,
|
||||||
|
output_type="str-all-except-first",
|
||||||
|
max_tokens=4096,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Agent 3: Market Risk Monitor
|
||||||
|
market_risk_agent = Agent(
|
||||||
|
agent_name="Market-Risk-Monitor",
|
||||||
|
agent_description="Monitors market conditions and identifies risk factors",
|
||||||
|
system_prompt="""You are a market risk monitor. Identify and assess:
|
||||||
|
- Market volatility trends
|
||||||
|
- Economic risk factors
|
||||||
|
- Geopolitical risks
|
||||||
|
- Interest rate risks
|
||||||
|
- Currency risks
|
||||||
|
|
||||||
|
Provide current risk alerts and trends.""",
|
||||||
|
max_loops=1,
|
||||||
|
model_name="gpt-4.1",
|
||||||
|
random_model_enabled=True,
|
||||||
|
dynamic_temperature_enabled=True,
|
||||||
|
output_type="str-all-except-first",
|
||||||
|
max_tokens=4096,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
swarm = SwarmRouter(
|
||||||
|
name="SwarmRouter",
|
||||||
|
description="A router that can route messages to the appropriate swarm",
|
||||||
|
agents=[
|
||||||
|
risk_metrics_agent,
|
||||||
|
portfolio_risk_agent,
|
||||||
|
],
|
||||||
|
max_loops=1,
|
||||||
|
swarm_type="SequentialWorkflow",
|
||||||
|
output_type="final",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
out = swarm.run(
|
||||||
|
"What are the best ways to short the EU markets. Give me specific tickrs to short and strategies to use. Create a comprehensive report with all the information you can find."
|
||||||
|
)
|
||||||
|
|
||||||
|
print(out)
|
||||||
@ -0,0 +1,243 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
SwarmRouter Performance Benchmark
|
||||||
|
|
||||||
|
This script benchmarks the performance improvements in SwarmRouter's _create_swarm method.
|
||||||
|
It compares the old O(n) elif chain vs the new O(1) factory pattern with caching.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
import statistics
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
from swarms.structs.swarm_router import SwarmRouter
|
||||||
|
from swarms.structs.agent import Agent
|
||||||
|
|
||||||
|
|
||||||
|
def create_mock_agents(num_agents: int = 3) -> List[Agent]:
|
||||||
|
"""Create mock agents for testing purposes."""
|
||||||
|
agents = []
|
||||||
|
for i in range(num_agents):
|
||||||
|
# Create a simple mock agent
|
||||||
|
agent = Agent(
|
||||||
|
agent_name=f"TestAgent_{i}",
|
||||||
|
system_prompt=f"You are test agent {i}",
|
||||||
|
model_name="gpt-4o-mini",
|
||||||
|
max_loops=1,
|
||||||
|
)
|
||||||
|
agents.append(agent)
|
||||||
|
return agents
|
||||||
|
|
||||||
|
|
||||||
|
def benchmark_swarm_creation(
|
||||||
|
swarm_types: List[str],
|
||||||
|
num_iterations: int = 100,
|
||||||
|
agents: List[Agent] = None,
|
||||||
|
) -> Dict[str, Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Benchmark swarm creation performance for different swarm types.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
swarm_types: List of swarm types to test
|
||||||
|
num_iterations: Number of iterations to run for each swarm type
|
||||||
|
agents: List of agents to use for testing
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary containing performance metrics for each swarm type
|
||||||
|
"""
|
||||||
|
if agents is None:
|
||||||
|
agents = create_mock_agents()
|
||||||
|
|
||||||
|
results = {}
|
||||||
|
|
||||||
|
for swarm_type in swarm_types:
|
||||||
|
print(f"Benchmarking {swarm_type}...")
|
||||||
|
times = []
|
||||||
|
|
||||||
|
for i in range(num_iterations):
|
||||||
|
# Create a fresh SwarmRouter instance for each test
|
||||||
|
router = SwarmRouter(
|
||||||
|
name=f"test-router-{i}",
|
||||||
|
agents=agents,
|
||||||
|
swarm_type=swarm_type,
|
||||||
|
telemetry_enabled=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Time the _create_swarm method
|
||||||
|
start_time = time.perf_counter()
|
||||||
|
try:
|
||||||
|
router._create_swarm(task="test task")
|
||||||
|
end_time = time.perf_counter()
|
||||||
|
times.append(end_time - start_time)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Failed to create {swarm_type}: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if times:
|
||||||
|
results[swarm_type] = {
|
||||||
|
"mean_time": statistics.mean(times),
|
||||||
|
"median_time": statistics.median(times),
|
||||||
|
"min_time": min(times),
|
||||||
|
"max_time": max(times),
|
||||||
|
"std_dev": (
|
||||||
|
statistics.stdev(times) if len(times) > 1 else 0
|
||||||
|
),
|
||||||
|
"total_iterations": len(times),
|
||||||
|
}
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def benchmark_caching_performance(
|
||||||
|
swarm_type: str = "SequentialWorkflow",
|
||||||
|
num_iterations: int = 50,
|
||||||
|
agents: List[Agent] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Benchmark the caching performance by creating the same swarm multiple times.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
swarm_type: The swarm type to test
|
||||||
|
num_iterations: Number of iterations to run
|
||||||
|
agents: List of agents to use for testing
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary containing caching performance metrics
|
||||||
|
"""
|
||||||
|
if agents is None:
|
||||||
|
agents = create_mock_agents()
|
||||||
|
|
||||||
|
print(f"Benchmarking caching performance for {swarm_type}...")
|
||||||
|
|
||||||
|
router = SwarmRouter(
|
||||||
|
name="cache-test-router",
|
||||||
|
agents=agents,
|
||||||
|
swarm_type=swarm_type,
|
||||||
|
telemetry_enabled=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
first_call_times = []
|
||||||
|
cached_call_times = []
|
||||||
|
|
||||||
|
for i in range(num_iterations):
|
||||||
|
# Clear cache for first call timing
|
||||||
|
router._swarm_cache.clear()
|
||||||
|
|
||||||
|
# Time first call (cache miss)
|
||||||
|
start_time = time.perf_counter()
|
||||||
|
router._create_swarm(task="test task", iteration=i)
|
||||||
|
end_time = time.perf_counter()
|
||||||
|
first_call_times.append(end_time - start_time)
|
||||||
|
|
||||||
|
# Time second call (cache hit)
|
||||||
|
start_time = time.perf_counter()
|
||||||
|
router._create_swarm(task="test task", iteration=i)
|
||||||
|
end_time = time.perf_counter()
|
||||||
|
cached_call_times.append(end_time - start_time)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"first_call_mean": statistics.mean(first_call_times),
|
||||||
|
"cached_call_mean": statistics.mean(cached_call_times),
|
||||||
|
"speedup_factor": statistics.mean(first_call_times)
|
||||||
|
/ statistics.mean(cached_call_times),
|
||||||
|
"cache_hit_ratio": 1.0, # 100% cache hit rate in this test
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def print_results(results: Dict[str, Dict[str, Any]]):
|
||||||
|
"""Print benchmark results in a formatted way."""
|
||||||
|
print("\n" + "=" * 60)
|
||||||
|
print("SWARM CREATION PERFORMANCE BENCHMARK RESULTS")
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
for swarm_type, metrics in results.items():
|
||||||
|
print(f"\n{swarm_type}:")
|
||||||
|
print(f" Mean time: {metrics['mean_time']:.6f} seconds")
|
||||||
|
print(f" Median time: {metrics['median_time']:.6f} seconds")
|
||||||
|
print(f" Min time: {metrics['min_time']:.6f} seconds")
|
||||||
|
print(f" Max time: {metrics['max_time']:.6f} seconds")
|
||||||
|
print(f" Std dev: {metrics['std_dev']:.6f} seconds")
|
||||||
|
print(f" Iterations: {metrics['total_iterations']}")
|
||||||
|
|
||||||
|
|
||||||
|
def print_caching_results(results: Dict[str, Any]):
|
||||||
|
"""Print caching benchmark results."""
|
||||||
|
print("\n" + "=" * 60)
|
||||||
|
print("CACHING PERFORMANCE BENCHMARK RESULTS")
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"First call mean time: {results['first_call_mean']:.6f} seconds"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"Cached call mean time: {results['cached_call_mean']:.6f} seconds"
|
||||||
|
)
|
||||||
|
print(f"Speedup factor: {results['speedup_factor']:.2f}x")
|
||||||
|
print(f"Cache hit ratio: {results['cache_hit_ratio']:.1%}")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Run the complete benchmark suite."""
|
||||||
|
print("SwarmRouter Performance Benchmark")
|
||||||
|
print(
|
||||||
|
"Testing O(1) factory pattern with caching vs O(n) elif chain"
|
||||||
|
)
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
# Create test agents
|
||||||
|
agents = create_mock_agents(3)
|
||||||
|
|
||||||
|
# Test different swarm types
|
||||||
|
swarm_types = [
|
||||||
|
"SequentialWorkflow",
|
||||||
|
"ConcurrentWorkflow",
|
||||||
|
"AgentRearrange",
|
||||||
|
"MixtureOfAgents",
|
||||||
|
"GroupChat",
|
||||||
|
"MultiAgentRouter",
|
||||||
|
"HeavySwarm",
|
||||||
|
"MALT",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Run creation benchmark
|
||||||
|
creation_results = benchmark_swarm_creation(
|
||||||
|
swarm_types=swarm_types[:4], # Test first 4 for speed
|
||||||
|
num_iterations=20,
|
||||||
|
agents=agents,
|
||||||
|
)
|
||||||
|
|
||||||
|
print_results(creation_results)
|
||||||
|
|
||||||
|
# Run caching benchmark
|
||||||
|
caching_results = benchmark_caching_performance(
|
||||||
|
swarm_type="SequentialWorkflow",
|
||||||
|
num_iterations=10,
|
||||||
|
agents=agents,
|
||||||
|
)
|
||||||
|
|
||||||
|
print_caching_results(caching_results)
|
||||||
|
|
||||||
|
# Calculate overall performance improvement
|
||||||
|
if creation_results:
|
||||||
|
avg_creation_time = statistics.mean(
|
||||||
|
[
|
||||||
|
metrics["mean_time"]
|
||||||
|
for metrics in creation_results.values()
|
||||||
|
]
|
||||||
|
)
|
||||||
|
print("\n" + "=" * 60)
|
||||||
|
print("PERFORMANCE SUMMARY")
|
||||||
|
print("=" * 60)
|
||||||
|
print(
|
||||||
|
f"Average swarm creation time: {avg_creation_time:.6f} seconds"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"Factory pattern provides O(1) lookup vs O(n) elif chain"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"Caching provides {caching_results['speedup_factor']:.2f}x speedup for repeated calls"
|
||||||
|
)
|
||||||
|
print("=" * 60)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -1,206 +0,0 @@
|
|||||||
# AI generate initial response
|
|
||||||
# AI decides how many "thinking rounds" it needs
|
|
||||||
# For each round:
|
|
||||||
# Generates 3 alternative responses
|
|
||||||
# Evaluates all responses
|
|
||||||
# Picks the best one
|
|
||||||
# Final response is the survivor of this AI battle royale
|
|
||||||
from swarms import Agent
|
|
||||||
|
|
||||||
|
|
||||||
# OpenAI function schema for determining thinking rounds
|
|
||||||
thinking_rounds_schema = {
|
|
||||||
"name": "determine_thinking_rounds",
|
|
||||||
"description": "Determines the optimal number of thinking rounds needed for a task",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"num_rounds": {
|
|
||||||
"type": "integer",
|
|
||||||
"description": "The number of thinking rounds needed (1-5)",
|
|
||||||
"minimum": 1,
|
|
||||||
"maximum": 5,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required": ["num_rounds"],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
# System prompt for determining thinking rounds
|
|
||||||
THINKING_ROUNDS_PROMPT = """You are an expert at determining the optimal number of thinking rounds needed for complex tasks. Your role is to analyze the task and determine how many rounds of thinking and evaluation would be most beneficial.
|
|
||||||
|
|
||||||
Consider the following factors when determining the number of rounds:
|
|
||||||
1. Task Complexity: More complex tasks may require more rounds
|
|
||||||
2. Potential for Multiple Valid Approaches: Tasks with multiple valid solutions need more rounds
|
|
||||||
3. Risk of Error: Higher-stakes tasks may benefit from more rounds
|
|
||||||
4. Time Sensitivity: Balance thoroughness with efficiency
|
|
||||||
|
|
||||||
Guidelines for number of rounds:
|
|
||||||
- 1 round: Simple, straightforward tasks with clear solutions
|
|
||||||
- 2-3 rounds: Moderately complex tasks with some ambiguity
|
|
||||||
- 4-5 rounds: Highly complex tasks with multiple valid approaches or high-stakes decisions
|
|
||||||
|
|
||||||
Your response should be a single number between 1 and 5, representing the optimal number of thinking rounds needed."""
|
|
||||||
|
|
||||||
# Schema for generating alternative responses
|
|
||||||
alternative_responses_schema = {
|
|
||||||
"name": "generate_alternatives",
|
|
||||||
"description": "Generates multiple alternative responses to a task",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"alternatives": {
|
|
||||||
"type": "array",
|
|
||||||
"description": "List of alternative responses",
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"response": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "The alternative response",
|
|
||||||
},
|
|
||||||
"reasoning": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Explanation of why this approach was chosen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["response", "reasoning"],
|
|
||||||
},
|
|
||||||
"minItems": 3,
|
|
||||||
"maxItems": 3,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required": ["alternatives"],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
# Schema for evaluating responses
|
|
||||||
evaluation_schema = {
|
|
||||||
"name": "evaluate_responses",
|
|
||||||
"description": "Evaluates and ranks alternative responses",
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"evaluation": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"best_response": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "The selected best response",
|
|
||||||
},
|
|
||||||
"ranking": {
|
|
||||||
"type": "array",
|
|
||||||
"description": "Ranked list of responses from best to worst",
|
|
||||||
"items": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"response": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "The response",
|
|
||||||
},
|
|
||||||
"score": {
|
|
||||||
"type": "number",
|
|
||||||
"description": "Score from 0-100",
|
|
||||||
},
|
|
||||||
"reasoning": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Explanation of the score",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": [
|
|
||||||
"response",
|
|
||||||
"score",
|
|
||||||
"reasoning",
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["best_response", "ranking"],
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required": ["evaluation"],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
# System prompt for generating alternatives
|
|
||||||
ALTERNATIVES_PROMPT = """You are an expert at generating diverse and creative alternative responses to tasks. Your role is to generate 3 distinct approaches to solving the given task.
|
|
||||||
|
|
||||||
For each alternative:
|
|
||||||
1. Consider a different perspective or approach
|
|
||||||
2. Provide clear reasoning for why this approach might be effective
|
|
||||||
3. Ensure alternatives are meaningfully different from each other
|
|
||||||
4. Maintain high quality and relevance to the task
|
|
||||||
|
|
||||||
Your response should include 3 alternatives, each with its own reasoning."""
|
|
||||||
|
|
||||||
# System prompt for evaluation
|
|
||||||
EVALUATION_PROMPT = """You are an expert at evaluating and comparing different responses to tasks. Your role is to critically analyze each response and determine which is the most effective.
|
|
||||||
|
|
||||||
Consider the following criteria when evaluating:
|
|
||||||
1. Relevance to the task
|
|
||||||
2. Completeness of the solution
|
|
||||||
3. Creativity and innovation
|
|
||||||
4. Practicality and feasibility
|
|
||||||
5. Clarity and coherence
|
|
||||||
|
|
||||||
Your response should include:
|
|
||||||
1. The best response selected
|
|
||||||
2. A ranked list of all responses with scores and reasoning"""
|
|
||||||
|
|
||||||
|
|
||||||
class CortAgent:
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
alternative_responses: int = 3,
|
|
||||||
):
|
|
||||||
self.thinking_rounds = Agent(
|
|
||||||
agent_name="CortAgent",
|
|
||||||
agent_description="CortAgent is a multi-step agent that uses a battle royale approach to determine the best response to a task.",
|
|
||||||
model_name="gpt-4o-mini",
|
|
||||||
max_loops=1,
|
|
||||||
dynamic_temperature_enabled=True,
|
|
||||||
tools_list_dictionary=thinking_rounds_schema,
|
|
||||||
system_prompt=THINKING_ROUNDS_PROMPT,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.alternatives_agent = Agent(
|
|
||||||
agent_name="CortAgentAlternatives",
|
|
||||||
agent_description="Generates multiple alternative responses to a task",
|
|
||||||
model_name="gpt-4o-mini",
|
|
||||||
max_loops=1,
|
|
||||||
dynamic_temperature_enabled=True,
|
|
||||||
tools_list_dictionary=alternative_responses_schema,
|
|
||||||
system_prompt=ALTERNATIVES_PROMPT,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.evaluation_agent = Agent(
|
|
||||||
agent_name="CortAgentEvaluation",
|
|
||||||
agent_description="Evaluates and ranks alternative responses",
|
|
||||||
model_name="gpt-4o-mini",
|
|
||||||
max_loops=1,
|
|
||||||
dynamic_temperature_enabled=True,
|
|
||||||
tools_list_dictionary=evaluation_schema,
|
|
||||||
system_prompt=EVALUATION_PROMPT,
|
|
||||||
)
|
|
||||||
|
|
||||||
def run(self, task: str):
|
|
||||||
# First determine number of thinking rounds
|
|
||||||
num_rounds = self.thinking_rounds.run(task)
|
|
||||||
|
|
||||||
# Initialize with the task
|
|
||||||
current_task = task
|
|
||||||
best_response = None
|
|
||||||
|
|
||||||
# Run the battle royale for the determined number of rounds
|
|
||||||
for round_num in range(num_rounds):
|
|
||||||
# Generate alternatives
|
|
||||||
alternatives = self.alternatives_agent.run(current_task)
|
|
||||||
|
|
||||||
# Evaluate alternatives
|
|
||||||
evaluation = self.evaluation_agent.run(alternatives)
|
|
||||||
|
|
||||||
# Update best response and current task for next round
|
|
||||||
best_response = evaluation["evaluation"]["best_response"]
|
|
||||||
current_task = f"Previous best response: {best_response}\nOriginal task: {task}"
|
|
||||||
|
|
||||||
return best_response
|
|
||||||
Loading…
Reference in new issue