diff --git a/api/Makefile b/api/Makefile new file mode 100644 index 00000000..dad1d750 --- /dev/null +++ b/api/Makefile @@ -0,0 +1,2 @@ +rerun: + bash ./rerun.sh diff --git a/api/boot.sh b/api/boot.sh index 4c56ae2a..eb48fea9 100644 --- a/api/boot.sh +++ b/api/boot.sh @@ -3,7 +3,7 @@ # to be run as swarms user set -e set -x -export ROOT="/mnt/data1/swarms" +export ROOT="" export HOME="${ROOT}/home/swarms" unset CONDA_EXE unset CONDA_PYTHON_EXE @@ -24,9 +24,9 @@ cd "${ROOT}/var/swarms/" pip install -e "${ROOT}/opt/swarms-memory" pip install "fastapi[standard]" pip install "loguru" +pip install "hunter" # for tracing pip install pydantic==2.8.2 - -# pip freeze +pip freeze # launch as systemd # python /opt/swarms/api/main.py diff --git a/api/install.sh b/api/install.sh index cd0b10c2..d6bb4d90 100644 --- a/api/install.sh +++ b/api/install.sh @@ -3,8 +3,9 @@ . ./.env # for secrets set -e # stop on any error set -x - -export ROOT="/mnt/data1/swarms" +export BRANCH="feature/ec2" +#export ROOT="/mnt/data1/swarms" +export ROOT="" # empty export WORKSOURCE="${ROOT}/opt/swarms/api" if [ ! -d "${ROOT}/opt/swarms/install/" ]; then @@ -12,14 +13,16 @@ if [ ! -d "${ROOT}/opt/swarms/install/" ]; then fi if [ ! -f "${ROOT}/opt/swarms/install/apt.txt" ]; then - sudo apt update - sudo apt install --allow-change-held-packages -y git virtualenv + apt update + apt install --allow-change-held-packages -y git virtualenv nginx expect + snap install aws-cli --classic echo 1 >"${ROOT}/opt/swarms/install/apt.txt" fi if [ ! -f "${ROOT}/opt/swarms/install/setup.txt" ]; then #rm -rf ./src/swarms # oops - adduser swarms --home "${ROOT}/home/swarms" || echo ignore + #adduser --disabled-password --comment "" swarms --home "${ROOT}/home/swarms" || echo ignore + adduser --disabled-password --gecos "" swarms --home "${ROOT}/home/swarms" || echo ignore git config --global --add safe.directory "${ROOT}/opt/swarms" git config --global --add safe.directory "${ROOT}/opt/swarms-memory" # we should have done this @@ -27,12 +30,13 @@ if [ ! -f "${ROOT}/opt/swarms/install/setup.txt" ]; then then git clone https://github.com/jmikedupont2/swarms "${ROOT}/opt/swarms/" fi - pushd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms" - git remote add local /time/2024/05/swarms/ || git remote set-url local /time/2024/05/swarms/ - git fetch local - git checkout feature/ec2 # switch branches - git pull local feature/ec2 - popd || exit 2 + cd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms" +# git remote add local /time/2024/05/swarms/ || git remote set-url local /time/2024/05/swarms/ +# git fetch local +# git stash + git checkout --force $BRANCH + git pull + git log -2 --patch | head -1000 if [ ! -d "${ROOT}/opt/swarms-memory/" ]; then git clone https://github.com/The-Swarm-Corporation/swarms-memory "${ROOT}/opt/swarms-memory" @@ -65,66 +69,55 @@ fi ## pull if [ ! -f "${ROOT}/opt/swarms/install/pull.txt" ]; then - pushd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms" - git fetch local - git checkout feature/ec2 # switch branches - git pull local feature/ec2 - popd || exit 2 - #echo 1 >"${ROOT}/opt/swarms/install/pull.txt" + cd "${ROOT}/opt/swarms/" || exit 1 # "we need swarms" +# git fetch local +# git stash + git checkout --force $BRANCH + git pull # $BRANCH + echo 1 >"${ROOT}/opt/swarms/install/pull.txt" fi - if [ ! -f "${ROOT}/opt/swarms/install/config.txt" ]; then - #WorkingDirectory=ROOT/var/run/swarms/ - # create the secrets mkdir -p "${ROOT}/var/run/swarms/secrets/" - echo "OPENAI_KEY=${OPENAI_KEY}" > "${ROOT}/var/run/swarms/secrets/env" - - # cache mkdir -p "${ROOT}/home/swarms/.cache/huggingface/hub" + # aws ssm get-parameter --name "swarms_openai_key" > /root/openaikey.txt + export OPENAI_KEY=`aws ssm get-parameter --name "swarms_openai_key" | jq .Parameter.Value -r ` + echo "OPENAI_KEY=${OPENAI_KEY}" > "${ROOT}/var/run/swarms/secrets/env" ## append new homedir - echo "TRANSFORMERS_CACHE=${ROOT}/home/swarms/.cache/huggingface/hub" >> "${ROOT}/var/run/swarms/secrets/env" + echo "HF_HOME=${ROOT}/home/swarms/.cache/huggingface/hub" >> "${ROOT}/var/run/swarms/secrets/env" echo "HOME=${ROOT}/home/swarms" >> "${ROOT}/var/run/swarms/secrets/env" + # attempt to move the workspace + echo 'WORKSPACE_DIR=${STATE_DIRECTORY}' >> "${ROOT}/var/run/swarms/secrets/env" #EnvironmentFile=ROOT/var/run/swarms/secrets/env #ExecStart=ROOT/var/run/uvicorn/env/bin/uvicorn \ # --uds ROOT/run/uvicorn/uvicorn-swarms-api.sock \ echo 1 >"${ROOT}/opt/swarms/install/config.txt" fi - -# if [ ! -f "${ROOT}/opt/swarms/install/uvicorn.txt" ]; then -# # we create a second installation of unicorn so agents cannot mess it up. -# mkdir -p "${ROOT}/var/run/uvicorn/env/" -# if [ ! -f "${ROOT}/var/run/uvicorn/env/" ]; -# then -# virtualenv "${ROOT}/var/run/uvicorn/env/" -# fi -# . "${ROOT}/var/run/uvicorn/env/bin/activate" -# pip install uvicorn -# echo 1 >"${ROOT}/opt/swarms/install/uvicorn.txt" -# fi if [ ! -f "${ROOT}/opt/swarms/install/nginx.txt" ]; then mkdir -p ${ROOT}/var/log/nginx/swarms/ fi -# reconfigure -# now we setup the service and replace root in the files -#echo cat "${WORKSOURCE}/nginx/site.conf" \| sed -e "s!ROOT!${ROOT}!g" -cat "${WORKSOURCE}/nginx/site.conf"| sed -e "s!ROOT!${ROOT}!g" > /etc/nginx/sites-enabled/default -#cat /etc/nginx/sites-enabled/default # create sock mkdir -p ${ROOT}/run/uvicorn/ chown -R swarms:swarms ${ROOT}/run/uvicorn +# reconfigure +# now we setup the service and replace root in the files +#echo cat "${WORKSOURCE}/nginx/site.conf" \| sed -e "s!ROOT!${ROOT}!g" +sed -e "s!ROOT!${ROOT}!g" > /etc/nginx/sites-enabled/default < "${WORKSOURCE}/nginx/site.conf" +#cat /etc/nginx/sites-enabled/default + # ROOT/var/run/swarms/uvicorn-swarms-api.sock; # access_log ROOT/var/log/nginx/swarms/access.log; # error_log ROOT/var/log/nginx/swarms/error.log; #echo cat "${WORKSOURCE}/systemd/uvicorn.service" \| sed -e "s!ROOT!/${ROOT}/!g" -cat "${WORKSOURCE}/systemd/uvicorn.service" | sed -e "s!ROOT!${ROOT}!g" > /etc/systemd/system/swarms-uvicorn.service - - +#cat "${WORKSOURCE}/systemd/uvicorn.service" +sed -e "s!ROOT!${ROOT}!g" > /etc/systemd/system/swarms-uvicorn.service < "${WORKSOURCE}/systemd/uvicorn.service" +grep . -h -n /etc/systemd/system/swarms-uvicorn.service + # if [ -f ${ROOT}/etc/systemd/system/swarms-uvicorn.service ]; # then # cp ${ROOT}/etc/systemd/system/swarms-uvicorn.service /etc/systemd/system/swarms-uvicorn.service @@ -139,14 +132,25 @@ cat "${WORKSOURCE}/systemd/uvicorn.service" | sed -e "s!ROOT!${ROOT}!g" > /etc/s #/run/uvicorn/ # triage chown -R swarms:swarms ${ROOT}/var/run/swarms/ +# Dec 12 10:55:50 mdupont-G470 unbuffer[3921723]: OSError: [Errno 30] Read-only file system: +#cat /etc/systemd/system/swarms-uvicorn.service +# now fix the perms +mkdir -p ${ROOT}/opt/swarms/api/agent_workspace/try_except_wrapper/ +chown -R swarms:swarms ${ROOT}/opt/swarms/api/ -#cat /etc/systemd/system/swarms-uvicorn.service # always reload systemctl daemon-reload # systemctl start swarms-uvicorn || systemctl status swarms-uvicorn.service && journalctl -xeu swarms-uvicorn.service systemctl start swarms-uvicorn || journalctl -xeu swarms-uvicorn.service # systemctl status swarms-uvicorn.service -# journalctl -xeu swarms-uvicorn.service +# journalctl -xeu swarms-uvicorn.serviceo systemctl enable swarms-uvicorn || journalctl -xeu swarms-uvicorn.service -service nginx restart +systemctl enable nginx +systemctl start nginx + +journalctl -xeu swarms-uvicorn.service | tail -200 || echo oops +systemctl status swarms-uvicorn.service || echo oops2 + +# mikes tools +apt install -y emacs-nox tmux diff --git a/api/main.py b/api/main.py index e71a150e..2175cdce 100644 --- a/api/main.py +++ b/api/main.py @@ -34,9 +34,14 @@ def foo(x): if k not in kind: print("KIND",x) kind[k]=1 - if "swarms" in m: - hunter.CallPrinter(x) + #hunter.CallPrinter(x) + print(x) + elif "uvicorn" in m: + #hunter.CallPrinter(x) + #print(x) + pass + if m not in seen: @@ -54,16 +59,6 @@ hunter.trace( # Load environment variables load_dotenv() -# # Configure Loguru -# logger.add( -# "logs/api_{time}.log", -# rotation="500 MB", -# retention="10 days", -# level="TRACE", -# format="{time} {level} {message}", -# backtrace=True, -# diagnose=True, -# ) class AgentStatus(str, Enum): """Enum for agent status.""" @@ -232,11 +227,12 @@ class AgentStore: ) # user_id -> [agent_ids] self.executor = ThreadPoolExecutor(max_workers=4) self._ensure_directories() - + logger.info(f"Created agent store: {self}") + def _ensure_directories(self): """Ensure required directories exist.""" - Path("logs").mkdir(exist_ok=True) - Path("states").mkdir(exist_ok=True) + #Path("logs").mkdir(exist_ok=True) + #Path("states").mkdir(exist_ok=True) def create_api_key(self, user_id: UUID, key_name: str) -> APIKey: """Create a new API key for a user.""" @@ -329,11 +325,9 @@ class AgentStore: "successful_completions": 0, } - # Add to user's agents list - if user_id not in self.user_agents: - self.user_agents[user_id] = [] - self.user_agents[user_id].append(agent_id) - + logger.info(f"Created agent with ID: {agent_id}") + logger.debug(f"Created agents:{self.agents.keys()}") + logger.info(f"agent store: {self}") return agent_id except Exception as e: @@ -345,6 +339,9 @@ class AgentStore: async def get_agent(self, agent_id: UUID) -> Agent: """Retrieve an agent by ID.""" + logger.info(f"agent store: {self}") + logger.info(f"Agent found: {agent_id}") + logger.debug(f"current agents:{self.agents.keys()}") agent = self.agents.get(agent_id) if not agent: logger.error(f"Agent not found: {agent_id}") @@ -882,7 +879,7 @@ app = create_app() if __name__ == '__main__': # freeze_support() - #print("yes in main") + print("yes in main") # Configure uvicorn logging logger.info("API Starting") @@ -895,6 +892,6 @@ if __name__ == '__main__': # reload=True, ) else: - #print("not in main") + print("not in main") pass diff --git a/api/nginx/site.conf b/api/nginx/site.conf index 1e40f0fb..5ec4d84b 100644 --- a/api/nginx/site.conf +++ b/api/nginx/site.conf @@ -9,6 +9,6 @@ server { add_header X-XSS-Protection "1; mode=block" always; location / { - proxy_pass http://unix:ROOT/var/run/swarms/uvicorn-swarms-api.sock; + proxy_pass http://127.0.0.1:5474; } } diff --git a/api/rerun-git.sh b/api/rerun-git.sh new file mode 100644 index 00000000..acae3684 --- /dev/null +++ b/api/rerun-git.sh @@ -0,0 +1,4 @@ +export ROOT=/mnt/data1/swarms; +git commit -m 'wip' +sudo rm "${ROOT}/opt/swarms/install/pull.txt" +sudo bash ./install.sh diff --git a/api/rerun.sh b/api/rerun.sh new file mode 100644 index 00000000..e12441b1 --- /dev/null +++ b/api/rerun.sh @@ -0,0 +1,6 @@ +export ROOT="" +#/mnt/data1/swarms; +#git commit -m 'wip' -a + +sudo rm ${ROOT}/opt/swarms/install/*; +sudo bash ./install.sh diff --git a/api/systemd/uvicorn.service b/api/systemd/uvicorn.service index 9fb4feb6..7618cf67 100644 --- a/api/systemd/uvicorn.service +++ b/api/systemd/uvicorn.service @@ -1,4 +1,4 @@ -# from https://github.com/encode/uvicorn/issues/678 +# derived from https://github.com/encode/uvicorn/issues/678 [Unit] Description=swarms After=network.target @@ -9,22 +9,26 @@ User=swarms Group=swarms DynamicUser=true WorkingDirectory=ROOT/opt/swarms/api/ +ReadWritePaths=ROOT/opt/swarms/api/agent_workspace/ +StateDirectory=swarms_state PrivateTmp=true +ProtectHome=true EnvironmentFile=ROOT/var/run/swarms/secrets/env # for debugging use -#ExecStart=/usr/bin/unbuffer ROOT/var/run/uvicorn/env/bin/uvicorn \ +#ExecStart=/usr/bin/unbuffer ROOT/var/run/uvicorn/env/bin/uvicorn -ExecStart=ROOT/var/run/uvicorn/env/bin/uvicorn \ +# this runs and is modifiable by the agent, so this can be destabilized +# FIXME: containerize the run(chroot jail?) +ExecStart=/usr/bin/unbuffer ROOT/var/swarms/agent_workspace/.venv/bin/uvicorn \ --proxy-headers \ --forwarded-allow-ips='*' \ --workers=4 \ - --port=54748 \ - --no-access-log \ - --uds ROOT/run/uvicorn/uvicorn-swarms-api.sock \ - _.asgi:application + --port=5474 \ + main:create_app + ExecReload=/bin/kill -HUP ${MAINPID} -RestartSec=1 +RestartSec=10 Restart=always [Install] diff --git a/names.sh b/names.sh new file mode 100644 index 00000000..03e82083 --- /dev/null +++ b/names.sh @@ -0,0 +1 @@ +grep -h -P -o "([a-zA-Z]+)" -r * |sort |uniq -c |sort -n >names.txt diff --git a/swarms/telemetry/capture_sys_data.py b/swarms/telemetry/capture_sys_data.py index d1f504a2..1d4f66d8 100644 --- a/swarms/telemetry/capture_sys_data.py +++ b/swarms/telemetry/capture_sys_data.py @@ -66,33 +66,27 @@ def log_agent_data(data_dict: dict) -> dict | None: if not data_dict: logger.error("Empty data dictionary provided") raise ValueError("data_dict cannot be empty") - -# url = "https://swarms.world/api/get-agents/log-agents" -# headers = { -# "Content-Type": "application/json", -# "Authorization": "Bearer sk-f24a13ed139f757d99cdd9cdcae710fccead92681606a97086d9711f69d44869", -# } - print(data_dict) + # url = "https://swarms.world/api/get-agents/log-agents" + # headers = { + # "Content-Type": "application/json", + # "Authorization": "Bearer sk-f24a13ed139f757d99cdd9cdcae710fccead92681606a97086d9711f69d44869", + # } + # print(data_dict) #try: # response = requests.post( # url, json=data_dict, headers=headers, timeout=10 # ) # response.raise_for_status() - # result = response.json() # return result - # except requests.exceptions.Timeout: # logger.warning("Request timed out") - # except requests.exceptions.HTTPError as e: # logger.error(f"HTTP error occurred: {e}") # if response.status_code == 401: # logger.error("Authentication failed - check API key") - # except requests.exceptions.RequestException as e: # logger.error(f"Error logging agent data: {e}") - #logger.error("Failed to log agent data") return {} diff --git a/swarms/utils/loguru_logger.py b/swarms/utils/loguru_logger.py index e36a1933..774510a3 100644 --- a/swarms/utils/loguru_logger.py +++ b/swarms/utils/loguru_logger.py @@ -1,38 +1,39 @@ import os import uuid +import sys # stderr from loguru import logger - def initialize_logger(log_folder: str = "logs"): - AGENT_WORKSPACE = "agent_workspace" + # AGENT_WORKSPACE = "agent_workspace" - # Check if WORKSPACE_DIR is set, if not, set it to AGENT_WORKSPACE - if "WORKSPACE_DIR" not in os.environ: - os.environ["WORKSPACE_DIR"] = AGENT_WORKSPACE + # # Check if WORKSPACE_DIR is set, if not, set it to AGENT_WORKSPACE + # if "WORKSPACE_DIR" not in os.environ: + # os.environ["WORKSPACE_DIR"] = AGENT_WORKSPACE - # Create a folder within the agent_workspace - log_folder_path = os.path.join( - os.getenv("WORKSPACE_DIR"), log_folder - ) - if not os.path.exists(log_folder_path): - os.makedirs(log_folder_path) + # # Create a folder within the agent_workspace + # log_folder_path = os.path.join( + # os.getenv("WORKSPACE_DIR"), log_folder + # ) + # if not os.path.exists(log_folder_path): + # os.makedirs(log_folder_path) - # Generate a unique identifier for the log file - uuid_for_log = str(uuid.uuid4()) - log_file_path = os.path.join( - #log_folder_path, f"{log_folder}_{uuid_for_log}.log" - log_folder_path, f"{log_folder}.log" - ) + # # Generate a unique identifier for the log file + # uuid_for_log = str(uuid.uuid4()) + # #log_file_path = os.path.join( + # #log_folder_path, f"{log_folder}_{uuid_for_log}.log" + # # log_folder_path, f"{log_folder}.log" + # #) - logger.add( - log_file_path, - level="TRACE", - colorize=True, - backtrace=True, - diagnose=True, - enqueue=True, - retention="10 days", - # compression="zip", - ) + # logger.add( + # # log_file_path, + # sys.stderr, + # level="TRACE", + # colorize=True, + # backtrace=True, + # diagnose=True, + # enqueue=True, + # retention="10 days", + # # compression="zip", + # ) return logger