Kye
1 year ago
parent
62f80ff949
commit
6c325a745e
File diff suppressed because one or more lines are too long
@ -1,3 +0,0 @@
|
||||
"""
|
||||
from swarms.apps import App
|
||||
"""
|
||||
@ -1,25 +0,0 @@
|
||||
# base App class
|
||||
class App:
|
||||
"""
|
||||
This is a base app class for examples
|
||||
|
||||
Args:
|
||||
worker: Worker Agent
|
||||
|
||||
Usage
|
||||
|
||||
app = App(Worker)
|
||||
app.run()
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
worker,
|
||||
):
|
||||
self.worker = worker
|
||||
self.worker.app = self
|
||||
|
||||
def run(self, task):
|
||||
"""Run the app"""
|
||||
pass
|
||||
@ -1,5 +1,13 @@
|
||||
from swarms.chunkers.base import BaseChunker
|
||||
from swarms.chunkers.chunk_seperator import ChunkSeparator
|
||||
from swarms.chunkers.markdown import MarkdownChunker
|
||||
from swarms.chunkers.text import TextChunker
|
||||
from swarms.chunkers.pdf import PdfChunker
|
||||
# from swarms.chunkers.base import BaseChunker
|
||||
# from swarms.chunkers.markdown import MarkdownChunker
|
||||
# from swarms.chunkers.text import TextChunker
|
||||
# from swarms.chunkers.pdf import PdfChunker
|
||||
|
||||
|
||||
# __all__ = [
|
||||
# "BaseChunker",
|
||||
# "ChunkSeparator",
|
||||
# "MarkdownChunker",
|
||||
# "TextChunker",
|
||||
# "PdfChunker",
|
||||
# ]
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
|
||||
_________ __ __ _____ __________ _____ _________
|
||||
/ _____// \ / \ / _ \ \______ \ / \ / _____/
|
||||
\_____ \ \ \/\/ // /_\ \ | _/ / \ / \ \_____ \
|
||||
/ \ \ // | \| | \/ Y \ / \
|
||||
/_______ / \__/\ / \____|__ /|____|_ /\____|__ //_______ /
|
||||
\/ \/ \/ \/ \/ \/
|
||||
@ -1,2 +1,7 @@
|
||||
from swarms.structs.workflow import Workflow
|
||||
from swarms.structs.task import Task
|
||||
|
||||
__all__ = [
|
||||
"Workflow",
|
||||
"Task",
|
||||
]
|
||||
|
||||
@ -1,2 +1,3 @@
|
||||
from swarms.utils.display_markdown import display_markdown_message
|
||||
from swarms.utils.futures import execute_futures_dict
|
||||
from swarms.utils.code_interpreter import SubprocessCodeInterpreter
|
||||
|
||||
@ -0,0 +1,162 @@
|
||||
import subprocess
|
||||
import threading
|
||||
import queue
|
||||
import time
|
||||
import traceback
|
||||
|
||||
|
||||
class BaseCodeInterpreter:
|
||||
"""
|
||||
.run is a generator that yields a dict with attributes: active_line, output
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def run(self, code):
|
||||
pass
|
||||
|
||||
def terminate(self):
|
||||
pass
|
||||
|
||||
|
||||
class SubprocessCodeInterpreter(BaseCodeInterpreter):
|
||||
"""
|
||||
SubprocessCodeinterpreter is a base class for code interpreters that run code in a subprocess.
|
||||
|
||||
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.start_cmd = ""
|
||||
self.process = None
|
||||
self.debug_mode = False
|
||||
self.output_queue = queue.Queue()
|
||||
self.done = threading.Event()
|
||||
|
||||
def detect_active_line(self, line):
|
||||
return None
|
||||
|
||||
def detect_end_of_execution(self, line):
|
||||
return None
|
||||
|
||||
def line_postprocessor(self, line):
|
||||
return line
|
||||
|
||||
def preprocess_code(self, code):
|
||||
"""
|
||||
This needs to insert an end_of_execution marker of some kind,
|
||||
which can be detected by detect_end_of_execution.
|
||||
|
||||
Optionally, add active line markers for detect_active_line.
|
||||
"""
|
||||
return code
|
||||
|
||||
def terminate(self):
|
||||
self.process.terminate()
|
||||
|
||||
def start_process(self):
|
||||
if self.process:
|
||||
self.terminate()
|
||||
|
||||
self.process = subprocess.Popen(
|
||||
self.start_cmd.split(),
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=0,
|
||||
universal_newlines=True,
|
||||
)
|
||||
threading.Thread(
|
||||
target=self.handle_stream_output,
|
||||
args=(self.process.stdout, False),
|
||||
daemon=True,
|
||||
).start()
|
||||
threading.Thread(
|
||||
target=self.handle_stream_output,
|
||||
args=(self.process.stderr, True),
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
def run(self, code):
|
||||
retry_count = 0
|
||||
max_retries = 3
|
||||
|
||||
# Setup
|
||||
try:
|
||||
code = self.preprocess_code(code)
|
||||
if not self.process:
|
||||
self.start_process()
|
||||
except:
|
||||
yield {"output": traceback.format_exc()}
|
||||
return
|
||||
|
||||
while retry_count <= max_retries:
|
||||
if self.debug_mode:
|
||||
print(f"Running code:\n{code}\n---")
|
||||
|
||||
self.done.clear()
|
||||
|
||||
try:
|
||||
self.process.stdin.write(code + "\n")
|
||||
self.process.stdin.flush()
|
||||
break
|
||||
except:
|
||||
if retry_count != 0:
|
||||
# For UX, I like to hide this if it happens once. Obviously feels better to not see errors
|
||||
# Most of the time it doesn't matter, but we should figure out why it happens frequently with:
|
||||
# applescript
|
||||
yield {"output": traceback.format_exc()}
|
||||
yield {"output": f"Retrying... ({retry_count}/{max_retries})"}
|
||||
yield {"output": "Restarting process."}
|
||||
|
||||
self.start_process()
|
||||
|
||||
retry_count += 1
|
||||
if retry_count > max_retries:
|
||||
yield {"output": "Maximum retries reached. Could not execute code."}
|
||||
return
|
||||
|
||||
while True:
|
||||
if not self.output_queue.empty():
|
||||
yield self.output_queue.get()
|
||||
else:
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
output = self.output_queue.get(timeout=0.3) # Waits for 0.3 seconds
|
||||
yield output
|
||||
except queue.Empty:
|
||||
if self.done.is_set():
|
||||
# Try to yank 3 more times from it... maybe there's something in there...
|
||||
# (I don't know if this actually helps. Maybe we just need to yank 1 more time)
|
||||
for _ in range(3):
|
||||
if not self.output_queue.empty():
|
||||
yield self.output_queue.get()
|
||||
time.sleep(0.2)
|
||||
break
|
||||
|
||||
def handle_stream_output(self, stream, is_error_stream):
|
||||
for line in iter(stream.readline, ""):
|
||||
if self.debug_mode:
|
||||
print(f"Received output line:\n{line}\n---")
|
||||
|
||||
line = self.line_postprocessor(line)
|
||||
|
||||
if line is None:
|
||||
continue # `line = None` is the postprocessor's signal to discard completely
|
||||
|
||||
if self.detect_active_line(line):
|
||||
active_line = self.detect_active_line(line)
|
||||
self.output_queue.put({"active_line": active_line})
|
||||
elif self.detect_end_of_execution(line):
|
||||
self.output_queue.put({"active_line": None})
|
||||
time.sleep(0.1)
|
||||
self.done.set()
|
||||
elif is_error_stream and "KeyboardInterrupt" in line:
|
||||
self.output_queue.put({"output": "KeyboardInterrupt"})
|
||||
time.sleep(0.1)
|
||||
self.done.set()
|
||||
else:
|
||||
self.output_queue.put({"output": line})
|
||||
@ -0,0 +1,23 @@
|
||||
from rich import print as rich_print
|
||||
from rich.markdown import Markdown
|
||||
from rich.rule import Rule
|
||||
|
||||
|
||||
def display_markdown_message(message):
|
||||
"""
|
||||
Display markdown message. Works with multiline strings with lots of indentation.
|
||||
Will automatically make single line > tags beautiful.
|
||||
"""
|
||||
|
||||
for line in message.split("\n"):
|
||||
line = line.strip()
|
||||
if line == "":
|
||||
print("")
|
||||
elif line == "---":
|
||||
rich_print(Rule(style="white"))
|
||||
else:
|
||||
rich_print(Markdown(line))
|
||||
|
||||
if "\n" not in message and message.startswith(">"):
|
||||
# Aesthetic choice. For these tags, they need a space below them
|
||||
print("")
|
||||
Loading…
Reference in new issue