Kye
10 months ago
parent
e38f48d3cc
commit
da7b33d819
@ -1,15 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "swarms-runtime" # The name of your project
|
|
||||||
version = "0.1.0" # The current version, adhering to semantic versioning
|
|
||||||
edition = "2021" # Specifies which edition of Rust you're using, e.g., 2018 or 2021
|
|
||||||
authors = ["Your Name <your.email@example.com>"] # Optional: specify the package authors
|
|
||||||
license = "MIT" # Optional: the license for your project
|
|
||||||
description = "A brief description of my project" # Optional: a short description of your project
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
cpython = "0.5"
|
|
||||||
rayon = "1.5"
|
|
||||||
|
|
||||||
[dependencies.pyo3]
|
|
||||||
version = "0.20.3"
|
|
||||||
features = ["extension-module", "auto-initialize"]
|
|
||||||
|
@ -0,0 +1,18 @@
|
|||||||
|
import os
|
||||||
|
from swarms import OpenAIChat, Agent
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
|
||||||
|
# Load environment variables
|
||||||
|
load_dotenv()
|
||||||
|
|
||||||
|
# Create a chat instance
|
||||||
|
llm = OpenAIChat(
|
||||||
|
api_key=os.getenv("OPENAI_API_KEY"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create an agent
|
||||||
|
agent = Agent(
|
||||||
|
agent_name="GPT-3",
|
||||||
|
llm=llm,
|
||||||
|
)
|
||||||
@ -0,0 +1,61 @@
|
|||||||
|
import os
|
||||||
|
from swarms import Gemini, Agent
|
||||||
|
from swarms.structs.multi_process_workflow import MultiProcessWorkflow
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
# Load the environment variables
|
||||||
|
load_dotenv()
|
||||||
|
|
||||||
|
# Gemini API key
|
||||||
|
api_key = os.getenv("GEMINI_API_KEY")
|
||||||
|
|
||||||
|
# Initialize LLM
|
||||||
|
llm = Gemini(
|
||||||
|
model_name="gemini-pro",
|
||||||
|
api_key=api_key,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Initialize the agents
|
||||||
|
finance_agent = Agent(
|
||||||
|
agent_name="Finance Agent",
|
||||||
|
llm=llm,
|
||||||
|
max_loops=1,
|
||||||
|
system_prompt="Finance",
|
||||||
|
)
|
||||||
|
|
||||||
|
marketing_agent = Agent(
|
||||||
|
agent_name="Marketing Agent",
|
||||||
|
llm=llm,
|
||||||
|
max_loops=1,
|
||||||
|
system_prompt="Marketing",
|
||||||
|
)
|
||||||
|
|
||||||
|
product_agent = Agent(
|
||||||
|
agent_name="Product Agent",
|
||||||
|
llm=llm,
|
||||||
|
max_loops=1,
|
||||||
|
system_prompt="Product",
|
||||||
|
)
|
||||||
|
|
||||||
|
other_agent = Agent(
|
||||||
|
agent_name="Other Agent",
|
||||||
|
llm=llm,
|
||||||
|
max_loops=1,
|
||||||
|
system_prompt="Other",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Swarm
|
||||||
|
workflow = MultiProcessWorkflow(
|
||||||
|
agents=[
|
||||||
|
finance_agent,
|
||||||
|
marketing_agent,
|
||||||
|
product_agent,
|
||||||
|
other_agent,
|
||||||
|
],
|
||||||
|
max_workers=5,
|
||||||
|
autosave=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Run the workflow
|
||||||
|
results = workflow.run("What")
|
||||||
@ -1,6 +1,4 @@
|
|||||||
# Import necessary libraries
|
|
||||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||||
|
|
||||||
from swarms import ToolAgent
|
from swarms import ToolAgent
|
||||||
|
|
||||||
# Load the pre-trained model and tokenizer
|
# Load the pre-trained model and tokenizer
|
||||||
@ -0,0 +1,183 @@
|
|||||||
|
import inspect
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from typing import Callable, List
|
||||||
|
|
||||||
|
from swarms.prompts.documentation import DOCUMENTATION_WRITER_SOP
|
||||||
|
from swarms import Agent, OpenAIChat
|
||||||
|
from swarms.utils.loguru_logger import logger
|
||||||
|
import concurrent
|
||||||
|
|
||||||
|
#########
|
||||||
|
from swarms.utils.file_processing import (
|
||||||
|
load_json,
|
||||||
|
sanitize_file_path,
|
||||||
|
zip_workspace,
|
||||||
|
create_file_in_folder,
|
||||||
|
zip_folders,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PythonDocumentationSwarm:
|
||||||
|
"""
|
||||||
|
A class for automating the documentation process for Python classes.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
agents (List[Agent]): A list of agents used for processing the documentation.
|
||||||
|
max_loops (int, optional): The maximum number of loops to run. Defaults to 4.
|
||||||
|
docs_module_name (str, optional): The name of the module where the documentation will be saved. Defaults to "swarms.structs".
|
||||||
|
docs_directory (str, optional): The directory where the documentation will be saved. Defaults to "docs/swarms/tokenizers".
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
agents (List[Agent]): A list of agents used for processing the documentation.
|
||||||
|
max_loops (int): The maximum number of loops to run.
|
||||||
|
docs_module_name (str): The name of the module where the documentation will be saved.
|
||||||
|
docs_directory (str): The directory where the documentation will be saved.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
agents: List[Agent],
|
||||||
|
max_loops: int = 4,
|
||||||
|
docs_module_name: str = "swarms.utils",
|
||||||
|
docs_directory: str = "docs/swarms/utils",
|
||||||
|
*args,
|
||||||
|
**kwargs,
|
||||||
|
):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.agents = agents
|
||||||
|
self.max_loops = max_loops
|
||||||
|
self.docs_module_name = docs_module_name
|
||||||
|
self.docs_directory = docs_directory
|
||||||
|
|
||||||
|
# Initialize agent name logging
|
||||||
|
logger.info(
|
||||||
|
"Agents used for documentation:"
|
||||||
|
f" {', '.join([agent.name for agent in agents])}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create the directory if it doesn't exist
|
||||||
|
dir_path = self.docs_directory
|
||||||
|
os.makedirs(dir_path, exist_ok=True)
|
||||||
|
logger.info(f"Documentation directory created at {dir_path}.")
|
||||||
|
|
||||||
|
def process_documentation(self, item):
|
||||||
|
"""
|
||||||
|
Process the documentation for a given class using OpenAI model and save it in a Markdown file.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
item: The class or function for which the documentation needs to be processed.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
doc = inspect.getdoc(item)
|
||||||
|
source = inspect.getsource(item)
|
||||||
|
is_class = inspect.isclass(item)
|
||||||
|
item_type = "Class Name" if is_class else "Name"
|
||||||
|
input_content = (
|
||||||
|
f"{item_type}:"
|
||||||
|
f" {item.__name__}\n\nDocumentation:\n{doc}\n\nSource"
|
||||||
|
f" Code:\n{source}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Process with OpenAI model (assuming the model's __call__ method takes this input and returns processed content)
|
||||||
|
for agent in self.agents:
|
||||||
|
processed_content = agent(
|
||||||
|
DOCUMENTATION_WRITER_SOP(
|
||||||
|
input_content, self.docs_module_name
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
doc_content = f"{processed_content}\n"
|
||||||
|
|
||||||
|
# Create the directory if it doesn't exist
|
||||||
|
dir_path = self.docs_directory
|
||||||
|
os.makedirs(dir_path, exist_ok=True)
|
||||||
|
|
||||||
|
# Write the processed documentation to a Markdown file
|
||||||
|
file_path = os.path.join(
|
||||||
|
dir_path, f"{item.__name__.lower()}.md"
|
||||||
|
)
|
||||||
|
with open(file_path, "w") as file:
|
||||||
|
file.write(doc_content)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"Documentation generated for {item.__name__}."
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(
|
||||||
|
f"Error processing documentation for {item.__name__}."
|
||||||
|
)
|
||||||
|
logger.error(e)
|
||||||
|
|
||||||
|
def run(self, python_items: List[Callable]):
|
||||||
|
"""
|
||||||
|
Run the documentation process for a list of Python items.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
python_items (List[Callable]): A list of Python classes or functions for which the documentation needs to be generated.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
threads = []
|
||||||
|
for item in python_items:
|
||||||
|
thread = threading.Thread(
|
||||||
|
target=self.process_documentation, args=(item,)
|
||||||
|
)
|
||||||
|
threads.append(thread)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
# Wait for all threads to complete
|
||||||
|
for thread in threads:
|
||||||
|
thread.join()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Documentation generated in 'swarms.structs'"
|
||||||
|
" directory."
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Error running documentation process.")
|
||||||
|
logger.error(e)
|
||||||
|
|
||||||
|
def run_concurrently(self, python_items: List[Callable]):
|
||||||
|
try:
|
||||||
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||||
|
executor.map(self.process_documentation, python_items)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Documentation generated in 'swarms.structs'"
|
||||||
|
" directory."
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Error running documentation process.")
|
||||||
|
logger.error(e)
|
||||||
|
|
||||||
|
|
||||||
|
# Example usage
|
||||||
|
# Initialize the agents
|
||||||
|
agent = Agent(
|
||||||
|
llm=OpenAIChat(max_tokens=3000),
|
||||||
|
agent_name="Documentation Agent",
|
||||||
|
system_prompt=(
|
||||||
|
"You write documentation for Python items functions and"
|
||||||
|
" classes, return in markdown"
|
||||||
|
),
|
||||||
|
max_loops=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Initialize the documentation swarm
|
||||||
|
doc_swarm = PythonDocumentationSwarm(
|
||||||
|
agents=[agent],
|
||||||
|
max_loops=1,
|
||||||
|
docs_module_name="swarms.structs",
|
||||||
|
docs_directory="docs/swarms/tokenizers",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Run the documentation process
|
||||||
|
doc_swarm.run(
|
||||||
|
[
|
||||||
|
load_json,
|
||||||
|
sanitize_file_path,
|
||||||
|
zip_workspace,
|
||||||
|
create_file_in_folder,
|
||||||
|
zip_folders,
|
||||||
|
]
|
||||||
|
)
|
||||||
@ -0,0 +1,21 @@
|
|||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
# Create a new directory for the log files if it doesn't exist
|
||||||
|
if not os.path.exists("artifacts"):
|
||||||
|
os.makedirs("artifacts")
|
||||||
|
|
||||||
|
# Walk through the current directory
|
||||||
|
for dirpath, dirnames, filenames in os.walk("."):
|
||||||
|
for filename in filenames:
|
||||||
|
# If the file is a log file
|
||||||
|
if filename.endswith(".log"):
|
||||||
|
# Construct the full file path
|
||||||
|
file_path = os.path.join(dirpath, filename)
|
||||||
|
# Move the log file to the 'artifacts' directory
|
||||||
|
shutil.move(file_path, "artifacts")
|
||||||
|
|
||||||
|
print(
|
||||||
|
"Moved all log files into the 'artifacts' directory and deleted"
|
||||||
|
" their original location."
|
||||||
|
)
|
||||||
@ -1,46 +0,0 @@
|
|||||||
import inspect
|
|
||||||
from typing import Callable
|
|
||||||
|
|
||||||
from termcolor import colored
|
|
||||||
|
|
||||||
|
|
||||||
def scrape_tool_func_docs(fn: Callable) -> str:
|
|
||||||
"""
|
|
||||||
Scrape the docstrings and parameters of a function decorated with `tool` and return a formatted string.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
fn (Callable): The function to scrape.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: A string containing the function's name, documentation string, and a list of its parameters. Each parameter is represented as a line containing the parameter's name, default value, and annotation.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
# If the function is a tool, get the original function
|
|
||||||
if hasattr(fn, "func"):
|
|
||||||
fn = fn.func
|
|
||||||
|
|
||||||
signature = inspect.signature(fn)
|
|
||||||
parameters = []
|
|
||||||
for name, param in signature.parameters.items():
|
|
||||||
parameters.append(
|
|
||||||
f"Name: {name}, Type:"
|
|
||||||
f" {param.default if param.default is not param.empty else 'None'},"
|
|
||||||
" Annotation:"
|
|
||||||
f" {param.annotation if param.annotation is not param.empty else 'None'}"
|
|
||||||
)
|
|
||||||
parameters_str = "\n".join(parameters)
|
|
||||||
return (
|
|
||||||
f"Function: {fn.__name__}\nDocstring:"
|
|
||||||
f" {inspect.getdoc(fn)}\nParameters:\n{parameters_str}"
|
|
||||||
)
|
|
||||||
except Exception as error:
|
|
||||||
print(
|
|
||||||
colored(
|
|
||||||
(
|
|
||||||
f"Error scraping tool function docs {error} try"
|
|
||||||
" optimizing your inputs with different"
|
|
||||||
" variables and attempt once more."
|
|
||||||
),
|
|
||||||
"red",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
Loading…
Reference in new issue