ai-mirow-projects
/
swarms
mirror of https://github.com/kyegomez/swarms
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source 
work in progress

now to test

we need to make sure the app is called correctly.

   main:create_app

in swarms/api/systemd/uvicorn.service

force pull

fixup

logging

wip

wip

wip

wip

wip

wip

wip

wip

wip

wip

wip

wip

adding scripts

bugfix

wip

wip

wip

wip

wip

no more sock

update

wip

wip

bugfix install useradd

switch branch

get rid of local

ignore error on user

more local

more local

bugfix APT

rerun

adding nginx

adding expect for unbuffer debugging

start nginx

bugfix print

hf_home and aws-cli
pull/699/head
mike dupont 4 months ago
parent e738b32259
commit 2a86aaf88f
11 changed files with 133 additions and 120 deletions
Show Stats Download Patch File Download Diff File

2
api/Makefile
Unescape View File

@ -0,0 +1,2 @@
rerun:
bash ./rerun.sh

6
api/boot.sh
Unescape View File

@ -3,7 +3,7 @@
# to be run as swarms user
set -e
set -x
export ROOT="/mnt/data1/swarms"
export ROOT=""
export HOME="${ROOT}/home/swarms"
unset CONDA_EXE
unset CONDA_PYTHON_EXE
@ -24,9 +24,9 @@ cd "${ROOT}/var/swarms/"
pip install -e "${ROOT}/opt/swarms-memory"
pip install "fastapi[standard]"
pip install "loguru"
pip install "hunter" # for tracing
pip install pydantic==2.8.2
# pip freeze
pip freeze
# launch as systemd
# python /opt/swarms/api/main.py

96
api/install.sh
Unescape View File

@ -3,8 +3,9 @@
. ./.env # for secrets
set -e # stop on any error
set -x
export ROOT="/mnt/data1/swarms"
export BRANCH="feature/ec2"
#export ROOT="/mnt/data1/swarms"
export ROOT="" # empty
export WORKSOURCE="${ROOT}/opt/swarms/api"
if [ ! -d "${ROOT}/opt/swarms/install/" ]; then
@ -12,14 +13,16 @@ if [ ! -d "${ROOT}/opt/swarms/install/" ]; then
fi
if [ ! -f "${ROOT}/opt/swarms/install/apt.txt" ]; then
sudo apt update
sudo apt install --allow-change-held-packages -y git virtualenv
apt update
apt install --allow-change-held-packages -y git virtualenv nginx expect
snap install aws-cli --classic
echo 1 >"${ROOT}/opt/swarms/install/apt.txt"
fi
if [ ! -f "${ROOT}/opt/swarms/install/setup.txt" ]; then
#rm -rf ./src/swarms # oops
adduser swarms --home "${ROOT}/home/swarms" || echo ignore
#adduser --disabled-password --comment "" swarms --home "${ROOT}/home/swarms" || echo ignore
adduser --disabled-password --gecos "" swarms --home "${ROOT}/home/swarms" || echo ignore
git config --global --add safe.directory "${ROOT}/opt/swarms"
git config --global --add safe.directory "${ROOT}/opt/swarms-memory"
# we should have done this
@ -27,12 +30,13 @@ if [ ! -f "${ROOT}/opt/swarms/install/setup.txt" ]; then
then
git clone https://github.com/jmikedupont2/swarms "${ROOT}/opt/swarms/"
fi
pushd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms"
git remote add local /time/2024/05/swarms/ || git remote set-url local /time/2024/05/swarms/
git fetch local
git checkout feature/ec2 # switch branches
git pull local feature/ec2
popd || exit 2
cd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms"
# git remote add local /time/2024/05/swarms/ || git remote set-url local /time/2024/05/swarms/
# git fetch local
# git stash
git checkout --force $BRANCH
git pull
git log -2 --patch | head -1000
if [ ! -d "${ROOT}/opt/swarms-memory/" ];
then
git clone https://github.com/The-Swarm-Corporation/swarms-memory "${ROOT}/opt/swarms-memory"
@ -65,65 +69,54 @@ fi
## pull
if [ ! -f "${ROOT}/opt/swarms/install/pull.txt" ]; then
pushd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms"
git fetch local
git checkout feature/ec2 # switch branches
git pull local feature/ec2
popd || exit 2
#echo 1 >"${ROOT}/opt/swarms/install/pull.txt"
cd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms"
# git fetch local
# git stash
git checkout --force $BRANCH
git pull # $BRANCH
echo 1 >"${ROOT}/opt/swarms/install/pull.txt"
fi
if [ ! -f "${ROOT}/opt/swarms/install/config.txt" ]; then
#WorkingDirectory=ROOT/var/run/swarms/
# create the secrets
mkdir -p "${ROOT}/var/run/swarms/secrets/"
echo "OPENAI_KEY=${OPENAI_KEY}" > "${ROOT}/var/run/swarms/secrets/env"
# cache
mkdir -p "${ROOT}/home/swarms/.cache/huggingface/hub"
# aws ssm get-parameter --name "swarms_openai_key" > /root/openaikey.txt
export OPENAI_KEY=`aws ssm get-parameter --name "swarms_openai_key" | jq .Parameter.Value -r `
echo "OPENAI_KEY=${OPENAI_KEY}" > "${ROOT}/var/run/swarms/secrets/env"
## append new homedir
echo "TRANSFORMERS_CACHE=${ROOT}/home/swarms/.cache/huggingface/hub" >> "${ROOT}/var/run/swarms/secrets/env"
echo "HF_HOME=${ROOT}/home/swarms/.cache/huggingface/hub" >> "${ROOT}/var/run/swarms/secrets/env"
echo "HOME=${ROOT}/home/swarms" >> "${ROOT}/var/run/swarms/secrets/env"
# attempt to move the workspace
echo 'WORKSPACE_DIR=${STATE_DIRECTORY}' >> "${ROOT}/var/run/swarms/secrets/env"
#EnvironmentFile=ROOT/var/run/swarms/secrets/env
#ExecStart=ROOT/var/run/uvicorn/env/bin/uvicorn \
# --uds ROOT/run/uvicorn/uvicorn-swarms-api.sock \
echo 1 >"${ROOT}/opt/swarms/install/config.txt"
fi
# if [ ! -f "${ROOT}/opt/swarms/install/uvicorn.txt" ]; then
# # we create a second installation of unicorn so agents cannot mess it up.
# mkdir -p "${ROOT}/var/run/uvicorn/env/"
# if [ ! -f "${ROOT}/var/run/uvicorn/env/" ];
# then
# virtualenv "${ROOT}/var/run/uvicorn/env/"
# fi
# . "${ROOT}/var/run/uvicorn/env/bin/activate"
# pip install uvicorn
# echo 1 >"${ROOT}/opt/swarms/install/uvicorn.txt"
# fi
if [ ! -f "${ROOT}/opt/swarms/install/nginx.txt" ]; then
mkdir -p ${ROOT}/var/log/nginx/swarms/
fi
# reconfigure
# now we setup the service and replace root in the files
#echo cat "${WORKSOURCE}/nginx/site.conf" \| sed -e "s!ROOT!${ROOT}!g"
cat "${WORKSOURCE}/nginx/site.conf"| sed -e "s!ROOT!${ROOT}!g" > /etc/nginx/sites-enabled/default
#cat /etc/nginx/sites-enabled/default
# create sock
mkdir -p ${ROOT}/run/uvicorn/
chown -R swarms:swarms ${ROOT}/run/uvicorn
# reconfigure
# now we setup the service and replace root in the files
#echo cat "${WORKSOURCE}/nginx/site.conf" \| sed -e "s!ROOT!${ROOT}!g"
sed -e "s!ROOT!${ROOT}!g" > /etc/nginx/sites-enabled/default < "${WORKSOURCE}/nginx/site.conf"
#cat /etc/nginx/sites-enabled/default
# ROOT/var/run/swarms/uvicorn-swarms-api.sock;
# access_log ROOT/var/log/nginx/swarms/access.log;
# error_log ROOT/var/log/nginx/swarms/error.log;
#echo cat "${WORKSOURCE}/systemd/uvicorn.service" \| sed -e "s!ROOT!/${ROOT}/!g"
cat "${WORKSOURCE}/systemd/uvicorn.service" | sed -e "s!ROOT!${ROOT}!g" > /etc/systemd/system/swarms-uvicorn.service
#cat "${WORKSOURCE}/systemd/uvicorn.service"
sed -e "s!ROOT!${ROOT}!g" > /etc/systemd/system/swarms-uvicorn.service < "${WORKSOURCE}/systemd/uvicorn.service"
grep . -h -n /etc/systemd/system/swarms-uvicorn.service
# if [ -f ${ROOT}/etc/systemd/system/swarms-uvicorn.service ];
# then
@ -139,14 +132,25 @@ cat "${WORKSOURCE}/systemd/uvicorn.service" | sed -e "s!ROOT!${ROOT}!g" > /etc/s
#/run/uvicorn/
# triage
chown -R swarms:swarms ${ROOT}/var/run/swarms/
# Dec 12 10:55:50 mdupont-G470 unbuffer[3921723]: OSError: [Errno 30] Read-only file system:
#cat /etc/systemd/system/swarms-uvicorn.service
# now fix the perms
mkdir -p ${ROOT}/opt/swarms/api/agent_workspace/try_except_wrapper/
chown -R swarms:swarms ${ROOT}/opt/swarms/api/
#cat /etc/systemd/system/swarms-uvicorn.service
# always reload
systemctl daemon-reload
# systemctl start swarms-uvicorn || systemctl status swarms-uvicorn.service && journalctl -xeu swarms-uvicorn.service
systemctl start swarms-uvicorn || journalctl -xeu swarms-uvicorn.service
# systemctl status swarms-uvicorn.service
# journalctl -xeu swarms-uvicorn.service
# journalctl -xeu swarms-uvicorn.serviceo
systemctl enable swarms-uvicorn || journalctl -xeu swarms-uvicorn.service
service nginx restart
systemctl enable nginx
systemctl start nginx
journalctl -xeu swarms-uvicorn.service | tail -200 || echo oops
systemctl status swarms-uvicorn.service || echo oops2
# mikes tools
apt install -y emacs-nox tmux

39
api/main.py
Unescape View File

@ -34,9 +34,14 @@ def foo(x):
if k not in kind:
print("KIND",x)
kind[k]=1
if "swarms" in m:
hunter.CallPrinter(x)
#hunter.CallPrinter(x)
print(x)
elif "uvicorn" in m:
#hunter.CallPrinter(x)
#print(x)
pass
if m not in seen:
@ -54,16 +59,6 @@ hunter.trace(
# Load environment variables
load_dotenv()
# # Configure Loguru
# logger.add(
# "logs/api_{time}.log",
# rotation="500 MB",
# retention="10 days",
# level="TRACE",
# format="{time} {level} {message}",
# backtrace=True,
# diagnose=True,
# )
class AgentStatus(str, Enum):
"""Enum for agent status."""
@ -232,11 +227,12 @@ class AgentStore:
) # user_id -> [agent_ids]
self.executor = ThreadPoolExecutor(max_workers=4)
self._ensure_directories()
logger.info(f"Created agent store: {self}")
def _ensure_directories(self):
"""Ensure required directories exist."""
Path("logs").mkdir(exist_ok=True)
Path("states").mkdir(exist_ok=True)
#Path("logs").mkdir(exist_ok=True)
#Path("states").mkdir(exist_ok=True)
def create_api_key(self, user_id: UUID, key_name: str) -> APIKey:
"""Create a new API key for a user."""
@ -329,11 +325,9 @@ class AgentStore:
"successful_completions": 0,
}
# Add to user's agents list
if user_id not in self.user_agents:
self.user_agents[user_id] = []
self.user_agents[user_id].append(agent_id)
logger.info(f"Created agent with ID: {agent_id}")
logger.debug(f"Created agents:{self.agents.keys()}")
logger.info(f"agent store: {self}")
return agent_id
except Exception as e:
@ -345,6 +339,9 @@ class AgentStore:
async def get_agent(self, agent_id: UUID) -> Agent:
"""Retrieve an agent by ID."""
logger.info(f"agent store: {self}")
logger.info(f"Agent found: {agent_id}")
logger.debug(f"current agents:{self.agents.keys()}")
agent = self.agents.get(agent_id)
if not agent:
logger.error(f"Agent not found: {agent_id}")
@ -882,7 +879,7 @@ app = create_app()
if __name__ == '__main__':
# freeze_support()
#print("yes in main")
print("yes in main")
# Configure uvicorn logging
logger.info("API Starting")
@ -895,6 +892,6 @@ if __name__ == '__main__':
# reload=True,
)
else:
#print("not in main")
print("not in main")
pass

2
api/nginx/site.conf
Unescape View File

@ -9,6 +9,6 @@ server {
add_header X-XSS-Protection "1; mode=block" always;
location / {
proxy_pass http://unix:ROOT/var/run/swarms/uvicorn-swarms-api.sock;
proxy_pass http://127.0.0.1:5474;
}
}

4
api/rerun-git.sh
Unescape View File

@ -0,0 +1,4 @@
export ROOT=/mnt/data1/swarms;
git commit -m 'wip'
sudo rm "${ROOT}/opt/swarms/install/pull.txt"
sudo bash ./install.sh

6
api/rerun.sh
Unescape View File

@ -0,0 +1,6 @@
export ROOT=""
#/mnt/data1/swarms;
#git commit -m 'wip' -a
sudo rm ${ROOT}/opt/swarms/install/*;
sudo bash ./install.sh

20
api/systemd/uvicorn.service
Unescape View File

@ -1,4 +1,4 @@
# from https://github.com/encode/uvicorn/issues/678
# derived from https://github.com/encode/uvicorn/issues/678
[Unit]
Description=swarms
After=network.target
@ -9,22 +9,26 @@ User=swarms
Group=swarms
DynamicUser=true
WorkingDirectory=ROOT/opt/swarms/api/
ReadWritePaths=ROOT/opt/swarms/api/agent_workspace/
StateDirectory=swarms_state
PrivateTmp=true
ProtectHome=true
EnvironmentFile=ROOT/var/run/swarms/secrets/env
# for debugging use
#ExecStart=/usr/bin/unbuffer ROOT/var/run/uvicorn/env/bin/uvicorn \
#ExecStart=/usr/bin/unbuffer ROOT/var/run/uvicorn/env/bin/uvicorn
ExecStart=ROOT/var/run/uvicorn/env/bin/uvicorn \
# this runs and is modifiable by the agent, so this can be destabilized
# FIXME: containerize the run(chroot jail?)
ExecStart=/usr/bin/unbuffer ROOT/var/swarms/agent_workspace/.venv/bin/uvicorn \
--proxy-headers \
--forwarded-allow-ips='*' \
--workers=4 \
--port=54748 \
--no-access-log \
--uds ROOT/run/uvicorn/uvicorn-swarms-api.sock \
_.asgi:application
--port=5474 \
main:create_app
ExecReload=/bin/kill -HUP ${MAINPID}
RestartSec=1
RestartSec=10
Restart=always
[Install]

1
names.sh
Unescape View File

@ -0,0 +1 @@
grep -h -P -o "([a-zA-Z]+)" -r * |sort |uniq -c |sort -n >names.txt

18
swarms/telemetry/capture_sys_data.py
Unescape View File

@ -66,33 +66,27 @@ def log_agent_data(data_dict: dict) -> dict | None:
if not data_dict:
logger.error("Empty data dictionary provided")
raise ValueError("data_dict cannot be empty")
# url = "https://swarms.world/api/get-agents/log-agents"
# headers = {
# "Content-Type": "application/json",
# "Authorization": "Bearer sk-f24a13ed139f757d99cdd9cdcae710fccead92681606a97086d9711f69d44869",
# }
print(data_dict)
# url = "https://swarms.world/api/get-agents/log-agents"
# headers = {
# "Content-Type": "application/json",
# "Authorization": "Bearer sk-f24a13ed139f757d99cdd9cdcae710fccead92681606a97086d9711f69d44869",
# }
# print(data_dict)
#try:
# response = requests.post(
# url, json=data_dict, headers=headers, timeout=10
# )
# response.raise_for_status()
# result = response.json()
# return result
# except requests.exceptions.Timeout:
# logger.warning("Request timed out")
# except requests.exceptions.HTTPError as e:
# logger.error(f"HTTP error occurred: {e}")
# if response.status_code == 401:
# logger.error("Authentication failed - check API key")
# except requests.exceptions.RequestException as e:
# logger.error(f"Error logging agent data: {e}")
#logger.error("Failed to log agent data")
return {}

55
swarms/utils/loguru_logger.py
Unescape View File

@ -1,38 +1,39 @@
import os
import uuid
import sys # stderr
from loguru import logger
def initialize_logger(log_folder: str = "logs"):
AGENT_WORKSPACE = "agent_workspace"
# AGENT_WORKSPACE = "agent_workspace"
# Check if WORKSPACE_DIR is set, if not, set it to AGENT_WORKSPACE
if "WORKSPACE_DIR" not in os.environ:
os.environ["WORKSPACE_DIR"] = AGENT_WORKSPACE
# # Check if WORKSPACE_DIR is set, if not, set it to AGENT_WORKSPACE
# if "WORKSPACE_DIR" not in os.environ:
# os.environ["WORKSPACE_DIR"] = AGENT_WORKSPACE
# Create a folder within the agent_workspace
log_folder_path = os.path.join(
os.getenv("WORKSPACE_DIR"), log_folder
)
if not os.path.exists(log_folder_path):
os.makedirs(log_folder_path)
# # Create a folder within the agent_workspace
# log_folder_path = os.path.join(
# os.getenv("WORKSPACE_DIR"), log_folder
# )
# if not os.path.exists(log_folder_path):
# os.makedirs(log_folder_path)
# Generate a unique identifier for the log file
uuid_for_log = str(uuid.uuid4())
log_file_path = os.path.join(
#log_folder_path, f"{log_folder}_{uuid_for_log}.log"
log_folder_path, f"{log_folder}.log"
)
# # Generate a unique identifier for the log file
# uuid_for_log = str(uuid.uuid4())
# #log_file_path = os.path.join(
# #log_folder_path, f"{log_folder}_{uuid_for_log}.log"
# # log_folder_path, f"{log_folder}.log"
# #)
logger.add(
log_file_path,
level="TRACE",
colorize=True,
backtrace=True,
diagnose=True,
enqueue=True,
retention="10 days",
# compression="zip",
)
# logger.add(
# # log_file_path,
# sys.stderr,
# level="TRACE",
# colorize=True,
# backtrace=True,
# diagnose=True,
# enqueue=True,
# retention="10 days",
# # compression="zip",
# )
return logger

Write Preview
Loading…
Cancel
Save