ai-mirow-projects
/
swarms
mirror of https://github.com/kyegomez/swarms
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source 
work in progress

now to test

we need to make sure the app is called correctly.

   main:create_app

in swarms/api/systemd/uvicorn.service

force pull

fixup

logging

wip

wip

wip

wip

wip

wip

wip

wip

wip

wip

wip

wip

adding scripts

bugfix

wip

wip

wip

wip

wip

no more sock

update

wip

wip

bugfix install useradd

switch branch

get rid of local

ignore error on user

more local

more local

bugfix APT

rerun

adding nginx

adding expect for unbuffer debugging

start nginx

bugfix print

hf_home and aws-cli
pull/699/head
mike dupont 4 months ago
parent e738b32259
commit 2a86aaf88f
11 changed files with 133 additions and 120 deletions
Show Stats Download Patch File Download Diff File

2
api/Makefile
Unescape View File

@ -0,0 +1,2 @@
rerun:
bash ./rerun.sh

6
api/boot.sh
Unescape View File

@ -3,7 +3,7 @@
# to be run as swarms user # to be run as swarms user
set -e set -e
set -x set -x
export ROOT="/mnt/data1/swarms" export ROOT=""
export HOME="${ROOT}/home/swarms" export HOME="${ROOT}/home/swarms"
unset CONDA_EXE unset CONDA_EXE
unset CONDA_PYTHON_EXE unset CONDA_PYTHON_EXE
@ -24,9 +24,9 @@ cd "${ROOT}/var/swarms/"
pip install -e "${ROOT}/opt/swarms-memory" pip install -e "${ROOT}/opt/swarms-memory"
pip install "fastapi[standard]" pip install "fastapi[standard]"
pip install "loguru" pip install "loguru"
pip install "hunter" # for tracing
pip install pydantic==2.8.2 pip install pydantic==2.8.2
pip freeze
# pip freeze
# launch as systemd # launch as systemd
# python /opt/swarms/api/main.py # python /opt/swarms/api/main.py

96
api/install.sh
Unescape View File

@ -3,8 +3,9 @@
. ./.env # for secrets . ./.env # for secrets
set -e # stop on any error set -e # stop on any error
set -x set -x
export BRANCH="feature/ec2"
export ROOT="/mnt/data1/swarms" #export ROOT="/mnt/data1/swarms"
export ROOT="" # empty
export WORKSOURCE="${ROOT}/opt/swarms/api" export WORKSOURCE="${ROOT}/opt/swarms/api"
if [ ! -d "${ROOT}/opt/swarms/install/" ]; then if [ ! -d "${ROOT}/opt/swarms/install/" ]; then
@ -12,14 +13,16 @@ if [ ! -d "${ROOT}/opt/swarms/install/" ]; then
fi fi
if [ ! -f "${ROOT}/opt/swarms/install/apt.txt" ]; then if [ ! -f "${ROOT}/opt/swarms/install/apt.txt" ]; then
sudo apt update apt update
sudo apt install --allow-change-held-packages -y git virtualenv apt install --allow-change-held-packages -y git virtualenv nginx expect
snap install aws-cli --classic
echo 1 >"${ROOT}/opt/swarms/install/apt.txt" echo 1 >"${ROOT}/opt/swarms/install/apt.txt"
fi fi
if [ ! -f "${ROOT}/opt/swarms/install/setup.txt" ]; then if [ ! -f "${ROOT}/opt/swarms/install/setup.txt" ]; then
#rm -rf ./src/swarms # oops #rm -rf ./src/swarms # oops
adduser swarms --home "${ROOT}/home/swarms" || echo ignore #adduser --disabled-password --comment "" swarms --home "${ROOT}/home/swarms" || echo ignore
adduser --disabled-password --gecos "" swarms --home "${ROOT}/home/swarms" || echo ignore
git config --global --add safe.directory "${ROOT}/opt/swarms" git config --global --add safe.directory "${ROOT}/opt/swarms"
git config --global --add safe.directory "${ROOT}/opt/swarms-memory" git config --global --add safe.directory "${ROOT}/opt/swarms-memory"
# we should have done this # we should have done this
@ -27,12 +30,13 @@ if [ ! -f "${ROOT}/opt/swarms/install/setup.txt" ]; then
then then
git clone https://github.com/jmikedupont2/swarms "${ROOT}/opt/swarms/" git clone https://github.com/jmikedupont2/swarms "${ROOT}/opt/swarms/"
fi fi
pushd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms" cd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms"
git remote add local /time/2024/05/swarms/ || git remote set-url local /time/2024/05/swarms/ # git remote add local /time/2024/05/swarms/ || git remote set-url local /time/2024/05/swarms/
git fetch local # git fetch local
git checkout feature/ec2 # switch branches # git stash
git pull local feature/ec2 git checkout --force $BRANCH
popd || exit 2 git pull
git log -2 --patch | head -1000
if [ ! -d "${ROOT}/opt/swarms-memory/" ]; if [ ! -d "${ROOT}/opt/swarms-memory/" ];
then then
git clone https://github.com/The-Swarm-Corporation/swarms-memory "${ROOT}/opt/swarms-memory" git clone https://github.com/The-Swarm-Corporation/swarms-memory "${ROOT}/opt/swarms-memory"
@ -65,65 +69,54 @@ fi
## pull ## pull
if [ ! -f "${ROOT}/opt/swarms/install/pull.txt" ]; then if [ ! -f "${ROOT}/opt/swarms/install/pull.txt" ]; then
pushd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms" cd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms"
git fetch local # git fetch local
git checkout feature/ec2 # switch branches # git stash
git pull local feature/ec2 git checkout --force $BRANCH
popd || exit 2 git pull # $BRANCH
#echo 1 >"${ROOT}/opt/swarms/install/pull.txt" echo 1 >"${ROOT}/opt/swarms/install/pull.txt"
fi fi
if [ ! -f "${ROOT}/opt/swarms/install/config.txt" ]; then if [ ! -f "${ROOT}/opt/swarms/install/config.txt" ]; then
#WorkingDirectory=ROOT/var/run/swarms/
# create the secrets
mkdir -p "${ROOT}/var/run/swarms/secrets/" mkdir -p "${ROOT}/var/run/swarms/secrets/"
echo "OPENAI_KEY=${OPENAI_KEY}" > "${ROOT}/var/run/swarms/secrets/env"
# cache
mkdir -p "${ROOT}/home/swarms/.cache/huggingface/hub" mkdir -p "${ROOT}/home/swarms/.cache/huggingface/hub"
# aws ssm get-parameter --name "swarms_openai_key" > /root/openaikey.txt
export OPENAI_KEY=`aws ssm get-parameter --name "swarms_openai_key" | jq .Parameter.Value -r `
echo "OPENAI_KEY=${OPENAI_KEY}" > "${ROOT}/var/run/swarms/secrets/env"
## append new homedir ## append new homedir
echo "TRANSFORMERS_CACHE=${ROOT}/home/swarms/.cache/huggingface/hub" >> "${ROOT}/var/run/swarms/secrets/env" echo "HF_HOME=${ROOT}/home/swarms/.cache/huggingface/hub" >> "${ROOT}/var/run/swarms/secrets/env"
echo "HOME=${ROOT}/home/swarms" >> "${ROOT}/var/run/swarms/secrets/env" echo "HOME=${ROOT}/home/swarms" >> "${ROOT}/var/run/swarms/secrets/env"
# attempt to move the workspace
echo 'WORKSPACE_DIR=${STATE_DIRECTORY}' >> "${ROOT}/var/run/swarms/secrets/env"
#EnvironmentFile=ROOT/var/run/swarms/secrets/env #EnvironmentFile=ROOT/var/run/swarms/secrets/env
#ExecStart=ROOT/var/run/uvicorn/env/bin/uvicorn \ #ExecStart=ROOT/var/run/uvicorn/env/bin/uvicorn \
# --uds ROOT/run/uvicorn/uvicorn-swarms-api.sock \ # --uds ROOT/run/uvicorn/uvicorn-swarms-api.sock \
echo 1 >"${ROOT}/opt/swarms/install/config.txt" echo 1 >"${ROOT}/opt/swarms/install/config.txt"
fi fi
# if [ ! -f "${ROOT}/opt/swarms/install/uvicorn.txt" ]; then
# # we create a second installation of unicorn so agents cannot mess it up.
# mkdir -p "${ROOT}/var/run/uvicorn/env/"
# if [ ! -f "${ROOT}/var/run/uvicorn/env/" ];
# then
# virtualenv "${ROOT}/var/run/uvicorn/env/"
# fi
# . "${ROOT}/var/run/uvicorn/env/bin/activate"
# pip install uvicorn
# echo 1 >"${ROOT}/opt/swarms/install/uvicorn.txt"
# fi
if [ ! -f "${ROOT}/opt/swarms/install/nginx.txt" ]; then if [ ! -f "${ROOT}/opt/swarms/install/nginx.txt" ]; then
mkdir -p ${ROOT}/var/log/nginx/swarms/ mkdir -p ${ROOT}/var/log/nginx/swarms/
fi fi
# reconfigure
# now we setup the service and replace root in the files
#echo cat "${WORKSOURCE}/nginx/site.conf" \| sed -e "s!ROOT!${ROOT}!g"
cat "${WORKSOURCE}/nginx/site.conf"| sed -e "s!ROOT!${ROOT}!g" > /etc/nginx/sites-enabled/default
#cat /etc/nginx/sites-enabled/default
# create sock # create sock
mkdir -p ${ROOT}/run/uvicorn/ mkdir -p ${ROOT}/run/uvicorn/
chown -R swarms:swarms ${ROOT}/run/uvicorn chown -R swarms:swarms ${ROOT}/run/uvicorn
# reconfigure
# now we setup the service and replace root in the files
#echo cat "${WORKSOURCE}/nginx/site.conf" \| sed -e "s!ROOT!${ROOT}!g"
sed -e "s!ROOT!${ROOT}!g" > /etc/nginx/sites-enabled/default < "${WORKSOURCE}/nginx/site.conf"
#cat /etc/nginx/sites-enabled/default
# ROOT/var/run/swarms/uvicorn-swarms-api.sock; # ROOT/var/run/swarms/uvicorn-swarms-api.sock;
# access_log ROOT/var/log/nginx/swarms/access.log; # access_log ROOT/var/log/nginx/swarms/access.log;
# error_log ROOT/var/log/nginx/swarms/error.log; # error_log ROOT/var/log/nginx/swarms/error.log;
#echo cat "${WORKSOURCE}/systemd/uvicorn.service" \| sed -e "s!ROOT!/${ROOT}/!g" #echo cat "${WORKSOURCE}/systemd/uvicorn.service" \| sed -e "s!ROOT!/${ROOT}/!g"
cat "${WORKSOURCE}/systemd/uvicorn.service" | sed -e "s!ROOT!${ROOT}!g" > /etc/systemd/system/swarms-uvicorn.service #cat "${WORKSOURCE}/systemd/uvicorn.service"
sed -e "s!ROOT!${ROOT}!g" > /etc/systemd/system/swarms-uvicorn.service < "${WORKSOURCE}/systemd/uvicorn.service"
grep . -h -n /etc/systemd/system/swarms-uvicorn.service
# if [ -f ${ROOT}/etc/systemd/system/swarms-uvicorn.service ]; # if [ -f ${ROOT}/etc/systemd/system/swarms-uvicorn.service ];
# then # then
@ -139,14 +132,25 @@ cat "${WORKSOURCE}/systemd/uvicorn.service" | sed -e "s!ROOT!${ROOT}!g" > /etc/s
#/run/uvicorn/ #/run/uvicorn/
# triage # triage
chown -R swarms:swarms ${ROOT}/var/run/swarms/ chown -R swarms:swarms ${ROOT}/var/run/swarms/
# Dec 12 10:55:50 mdupont-G470 unbuffer[3921723]: OSError: [Errno 30] Read-only file system:
#cat /etc/systemd/system/swarms-uvicorn.service
# now fix the perms
mkdir -p ${ROOT}/opt/swarms/api/agent_workspace/try_except_wrapper/
chown -R swarms:swarms ${ROOT}/opt/swarms/api/
#cat /etc/systemd/system/swarms-uvicorn.service
# always reload # always reload
systemctl daemon-reload systemctl daemon-reload
# systemctl start swarms-uvicorn || systemctl status swarms-uvicorn.service && journalctl -xeu swarms-uvicorn.service # systemctl start swarms-uvicorn || systemctl status swarms-uvicorn.service && journalctl -xeu swarms-uvicorn.service
systemctl start swarms-uvicorn || journalctl -xeu swarms-uvicorn.service systemctl start swarms-uvicorn || journalctl -xeu swarms-uvicorn.service
# systemctl status swarms-uvicorn.service # systemctl status swarms-uvicorn.service
# journalctl -xeu swarms-uvicorn.service # journalctl -xeu swarms-uvicorn.serviceo
systemctl enable swarms-uvicorn || journalctl -xeu swarms-uvicorn.service systemctl enable swarms-uvicorn || journalctl -xeu swarms-uvicorn.service
service nginx restart systemctl enable nginx
systemctl start nginx
journalctl -xeu swarms-uvicorn.service | tail -200 || echo oops
systemctl status swarms-uvicorn.service || echo oops2
# mikes tools
apt install -y emacs-nox tmux

39
api/main.py
Unescape View File

@ -34,9 +34,14 @@ def foo(x):
if k not in kind: if k not in kind:
print("KIND",x) print("KIND",x)
kind[k]=1 kind[k]=1
if "swarms" in m: if "swarms" in m:
hunter.CallPrinter(x) #hunter.CallPrinter(x)
print(x)
elif "uvicorn" in m:
#hunter.CallPrinter(x)
#print(x)
pass
if m not in seen: if m not in seen:
@ -54,16 +59,6 @@ hunter.trace(
# Load environment variables # Load environment variables
load_dotenv() load_dotenv()
# # Configure Loguru
# logger.add(
# "logs/api_{time}.log",
# rotation="500 MB",
# retention="10 days",
# level="TRACE",
# format="{time} {level} {message}",
# backtrace=True,
# diagnose=True,
# )
class AgentStatus(str, Enum): class AgentStatus(str, Enum):
"""Enum for agent status.""" """Enum for agent status."""
@ -232,11 +227,12 @@ class AgentStore:
) # user_id -> [agent_ids] ) # user_id -> [agent_ids]
self.executor = ThreadPoolExecutor(max_workers=4) self.executor = ThreadPoolExecutor(max_workers=4)
self._ensure_directories() self._ensure_directories()
logger.info(f"Created agent store: {self}")
def _ensure_directories(self): def _ensure_directories(self):
"""Ensure required directories exist.""" """Ensure required directories exist."""
Path("logs").mkdir(exist_ok=True) #Path("logs").mkdir(exist_ok=True)
Path("states").mkdir(exist_ok=True) #Path("states").mkdir(exist_ok=True)
def create_api_key(self, user_id: UUID, key_name: str) -> APIKey: def create_api_key(self, user_id: UUID, key_name: str) -> APIKey:
"""Create a new API key for a user.""" """Create a new API key for a user."""
@ -329,11 +325,9 @@ class AgentStore:
"successful_completions": 0, "successful_completions": 0,
} }
# Add to user's agents list logger.info(f"Created agent with ID: {agent_id}")
if user_id not in self.user_agents: logger.debug(f"Created agents:{self.agents.keys()}")
self.user_agents[user_id] = [] logger.info(f"agent store: {self}")
self.user_agents[user_id].append(agent_id)
return agent_id return agent_id
except Exception as e: except Exception as e:
@ -345,6 +339,9 @@ class AgentStore:
async def get_agent(self, agent_id: UUID) -> Agent: async def get_agent(self, agent_id: UUID) -> Agent:
"""Retrieve an agent by ID.""" """Retrieve an agent by ID."""
logger.info(f"agent store: {self}")
logger.info(f"Agent found: {agent_id}")
logger.debug(f"current agents:{self.agents.keys()}")
agent = self.agents.get(agent_id) agent = self.agents.get(agent_id)
if not agent: if not agent:
logger.error(f"Agent not found: {agent_id}") logger.error(f"Agent not found: {agent_id}")
@ -882,7 +879,7 @@ app = create_app()
if __name__ == '__main__': if __name__ == '__main__':
# freeze_support() # freeze_support()
#print("yes in main") print("yes in main")
# Configure uvicorn logging # Configure uvicorn logging
logger.info("API Starting") logger.info("API Starting")
@ -895,6 +892,6 @@ if __name__ == '__main__':
# reload=True, # reload=True,
) )
else: else:
#print("not in main") print("not in main")
pass pass

2
api/nginx/site.conf
Unescape View File

@ -9,6 +9,6 @@ server {
add_header X-XSS-Protection "1; mode=block" always; add_header X-XSS-Protection "1; mode=block" always;
location / { location / {
proxy_pass http://unix:ROOT/var/run/swarms/uvicorn-swarms-api.sock; proxy_pass http://127.0.0.1:5474;
} }
} }

4
api/rerun-git.sh
Unescape View File

@ -0,0 +1,4 @@
export ROOT=/mnt/data1/swarms;
git commit -m 'wip'
sudo rm "${ROOT}/opt/swarms/install/pull.txt"
sudo bash ./install.sh

6
api/rerun.sh
Unescape View File

@ -0,0 +1,6 @@
export ROOT=""
#/mnt/data1/swarms;
#git commit -m 'wip' -a
sudo rm ${ROOT}/opt/swarms/install/*;
sudo bash ./install.sh

20
api/systemd/uvicorn.service
Unescape View File

@ -1,4 +1,4 @@
# from https://github.com/encode/uvicorn/issues/678 # derived from https://github.com/encode/uvicorn/issues/678
[Unit] [Unit]
Description=swarms Description=swarms
After=network.target After=network.target
@ -9,22 +9,26 @@ User=swarms
Group=swarms Group=swarms
DynamicUser=true DynamicUser=true
WorkingDirectory=ROOT/opt/swarms/api/ WorkingDirectory=ROOT/opt/swarms/api/
ReadWritePaths=ROOT/opt/swarms/api/agent_workspace/
StateDirectory=swarms_state
PrivateTmp=true PrivateTmp=true
ProtectHome=true
EnvironmentFile=ROOT/var/run/swarms/secrets/env EnvironmentFile=ROOT/var/run/swarms/secrets/env
# for debugging use # for debugging use
#ExecStart=/usr/bin/unbuffer ROOT/var/run/uvicorn/env/bin/uvicorn \ #ExecStart=/usr/bin/unbuffer ROOT/var/run/uvicorn/env/bin/uvicorn
ExecStart=ROOT/var/run/uvicorn/env/bin/uvicorn \ # this runs and is modifiable by the agent, so this can be destabilized
# FIXME: containerize the run(chroot jail?)
ExecStart=/usr/bin/unbuffer ROOT/var/swarms/agent_workspace/.venv/bin/uvicorn \
--proxy-headers \ --proxy-headers \
--forwarded-allow-ips='*' \ --forwarded-allow-ips='*' \
--workers=4 \ --workers=4 \
--port=54748 \ --port=5474 \
--no-access-log \ main:create_app
--uds ROOT/run/uvicorn/uvicorn-swarms-api.sock \
_.asgi:application
ExecReload=/bin/kill -HUP ${MAINPID} ExecReload=/bin/kill -HUP ${MAINPID}
RestartSec=1 RestartSec=10
Restart=always Restart=always
[Install] [Install]

1
names.sh
Unescape View File

@ -0,0 +1 @@
grep -h -P -o "([a-zA-Z]+)" -r * |sort |uniq -c |sort -n >names.txt

18
swarms/telemetry/capture_sys_data.py
Unescape View File

@ -66,33 +66,27 @@ def log_agent_data(data_dict: dict) -> dict | None:
if not data_dict: if not data_dict:
logger.error("Empty data dictionary provided") logger.error("Empty data dictionary provided")
raise ValueError("data_dict cannot be empty") raise ValueError("data_dict cannot be empty")
# url = "https://swarms.world/api/get-agents/log-agents"
# headers = {
# "Content-Type": "application/json",
# "Authorization": "Bearer sk-f24a13ed139f757d99cdd9cdcae710fccead92681606a97086d9711f69d44869",
# }
print(data_dict) print(data_dict)
# url = "https://swarms.world/api/get-agents/log-agents"
# headers = {
# "Content-Type": "application/json",
# "Authorization": "Bearer sk-f24a13ed139f757d99cdd9cdcae710fccead92681606a97086d9711f69d44869",
# }
# print(data_dict)
#try: #try:
# response = requests.post( # response = requests.post(
# url, json=data_dict, headers=headers, timeout=10 # url, json=data_dict, headers=headers, timeout=10
# ) # )
# response.raise_for_status() # response.raise_for_status()
# result = response.json() # result = response.json()
# return result # return result
# except requests.exceptions.Timeout: # except requests.exceptions.Timeout:
# logger.warning("Request timed out") # logger.warning("Request timed out")
# except requests.exceptions.HTTPError as e: # except requests.exceptions.HTTPError as e:
# logger.error(f"HTTP error occurred: {e}") # logger.error(f"HTTP error occurred: {e}")
# if response.status_code == 401: # if response.status_code == 401:
# logger.error("Authentication failed - check API key") # logger.error("Authentication failed - check API key")
# except requests.exceptions.RequestException as e: # except requests.exceptions.RequestException as e:
# logger.error(f"Error logging agent data: {e}") # logger.error(f"Error logging agent data: {e}")
#logger.error("Failed to log agent data") #logger.error("Failed to log agent data")
return {} return {}

55
swarms/utils/loguru_logger.py
Unescape View File

@ -1,38 +1,39 @@
import os import os
import uuid import uuid
import sys # stderr
from loguru import logger from loguru import logger
def initialize_logger(log_folder: str = "logs"): def initialize_logger(log_folder: str = "logs"):
AGENT_WORKSPACE = "agent_workspace" # AGENT_WORKSPACE = "agent_workspace"
# Check if WORKSPACE_DIR is set, if not, set it to AGENT_WORKSPACE # # Check if WORKSPACE_DIR is set, if not, set it to AGENT_WORKSPACE
if "WORKSPACE_DIR" not in os.environ: # if "WORKSPACE_DIR" not in os.environ:
os.environ["WORKSPACE_DIR"] = AGENT_WORKSPACE # os.environ["WORKSPACE_DIR"] = AGENT_WORKSPACE
# Create a folder within the agent_workspace # # Create a folder within the agent_workspace
log_folder_path = os.path.join( # log_folder_path = os.path.join(
os.getenv("WORKSPACE_DIR"), log_folder # os.getenv("WORKSPACE_DIR"), log_folder
) # )
if not os.path.exists(log_folder_path): # if not os.path.exists(log_folder_path):
os.makedirs(log_folder_path) # os.makedirs(log_folder_path)
# Generate a unique identifier for the log file # # Generate a unique identifier for the log file
uuid_for_log = str(uuid.uuid4()) # uuid_for_log = str(uuid.uuid4())
log_file_path = os.path.join( # #log_file_path = os.path.join(
#log_folder_path, f"{log_folder}_{uuid_for_log}.log" # #log_folder_path, f"{log_folder}_{uuid_for_log}.log"
log_folder_path, f"{log_folder}.log" # # log_folder_path, f"{log_folder}.log"
) # #)
logger.add( # logger.add(
log_file_path, # # log_file_path,
level="TRACE", # sys.stderr,
colorize=True, # level="TRACE",
backtrace=True, # colorize=True,
diagnose=True, # backtrace=True,
enqueue=True, # diagnose=True,
retention="10 days", # enqueue=True,
# compression="zip", # retention="10 days",
) # # compression="zip",
# )
return logger return logger

Write Preview
Loading…
Cancel
Save