From d6f662246316068438bc3fd579b6f3991fb7b2f5 Mon Sep 17 00:00:00 2001 From: Zohaib Rauf Date: Sat, 10 Feb 2024 18:52:25 -0800 Subject: [PATCH 1/3] Added NGrok --- OS/01/device.py | 28 ++++++++++++++++++---------- OS/01/requirements.txt | 26 +++++++++++++++----------- OS/01/server.py | 27 +++++++++++++++++++++------ OS/01/start.sh | 28 +++++++++++++++++++++------- 4 files changed, 75 insertions(+), 34 deletions(-) diff --git a/OS/01/device.py b/OS/01/device.py index 8c4ef89..c9a8e7d 100644 --- a/OS/01/device.py +++ b/OS/01/device.py @@ -7,10 +7,8 @@ from starlette.websockets import WebSocket from queue import Queue from pynput import keyboard import json -import traceback import websockets import queue -import pydub import ast from pydub import AudioSegment from pydub.playback import play @@ -22,6 +20,7 @@ from datetime import datetime from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run? from utils.kernel import put_kernel_messages_into_queue from stt import stt_wav +import asyncio # Configure logging logging.basicConfig(format='%(message)s', level=logging.getLevelName(os.getenv('DEBUG_LEVEL', 'INFO').upper())) @@ -34,10 +33,6 @@ RATE = 44100 # Sample rate RECORDING = False # Flag to control recording state SPACEBAR_PRESSED = False # Flag to track spacebar press state -# Configuration for WebSocket -WS_URL = os.getenv('SERVER_URL') -if not WS_URL: - raise ValueError("The environment variable SERVER_URL is not set. Please set it to proceed.") # Initialize PyAudio p = pyaudio.PyAudio() @@ -127,7 +122,6 @@ def on_release(key): logging.info("Exiting...") os._exit(0) -import asyncio send_queue = queue.Queue() @@ -140,7 +134,8 @@ async def message_sender(websocket): async def websocket_communication(WS_URL): while True: try: - async with websockets.connect(WS_URL) as websocket: + headers = {"ngrok-skip-browser-warning": str(80)} if os.getenv('NGROK_AUTHTOKEN') else {} + async with websockets.connect(WS_URL, extra_headers=headers) as websocket: logging.info("Press the spacebar to start/stop recording. Press ESC to exit.") asyncio.create_task(message_sender(websocket)) @@ -185,14 +180,27 @@ async def websocket_communication(WS_URL): result = interpreter.computer.run(language, code) send_queue.put(result) - except: - # traceback.print_exc() + except Exception as e: + logging.exception(f"An error occurred during websocket communication. {e}") logging.info(f"Connecting to `{WS_URL}`...") await asyncio.sleep(2) if __name__ == "__main__": + # Configuration for WebSocket async def main(): + WS_URL = os.getenv('SERVER_CONNECTION_URL') + + start_time = time.time() + while WS_URL is None: + if time.time() - start_time > 60: + logging.error("SERVER_CONNECTION_URL environment variable is not set after 1 minute. Exiting...") + raise ValueError("The environment variable SERVER_CONNECTION_URL is not set. Please set it to proceed.") + + logging.info("SERVER_CONNECTION_URL environment variable is not set. Waiting...") + await asyncio.sleep(5) # Wait for 5 seconds before checking again + WS_URL = os.getenv('SERVER_CONNECTION_URL') + # Start the WebSocket communication asyncio.create_task(websocket_communication(WS_URL)) diff --git a/OS/01/requirements.txt b/OS/01/requirements.txt index 6e70eb4..08d5d35 100644 --- a/OS/01/requirements.txt +++ b/OS/01/requirements.txt @@ -1,12 +1,16 @@ git+https://github.com/KillianLucas/open-interpreter.git -asyncio -PyAudio -pynput -fastapi -uvicorn -websockets -playsound -python-dotenv -ffmpeg-python -textual -pydub \ No newline at end of file +asyncio==3.4.3 +PyAudio==0.2.14 +pynput==1.7.6 +fastapi==0.109.2 +uvicorn==0.27.1 +websockets==12.0 +playsound==1.3.0 +python-dotenv==1.0.1 +ffmpeg-python==0.2.0 +textual==0.50.1 +pydub==0.25.1 +ngrok==1.0.0 +wheel + + diff --git a/OS/01/server.py b/OS/01/server.py index 45ea07b..37b56a4 100644 --- a/OS/01/server.py +++ b/OS/01/server.py @@ -1,18 +1,13 @@ from starlette.websockets import WebSocketDisconnect import ast import json -import time import queue import os import logging import traceback -from queue import Queue -from threading import Thread -import threading -import uvicorn import re from fastapi import FastAPI -from threading import Thread +from fastapi.responses import PlainTextResponse from starlette.websockets import WebSocket from stt import stt_bytes from tts import tts @@ -22,6 +17,8 @@ import urllib.parse from utils.kernel import put_kernel_messages_into_queue from i import configure_interpreter from interpreter import interpreter +import ngrok +import signal # Configure logging logging.basicConfig(format='%(message)s', level=logging.getLevelName(os.getenv('DEBUG_LEVEL', 'INFO').upper())) @@ -89,6 +86,10 @@ if os.getenv('CODE_RUNNER') == "device": # Configure interpreter interpreter = configure_interpreter(interpreter) +@app.get("/ping") +async def ping(): + return PlainTextResponse("pong") + @app.websocket("/") async def websocket_endpoint(websocket: WebSocket): await websocket.accept() @@ -232,6 +233,7 @@ from uvicorn import Config, Server if __name__ == "__main__": async def main(): + # Start listening asyncio.create_task(listener()) @@ -243,6 +245,19 @@ if __name__ == "__main__": if not server_url: raise ValueError("The environment variable SERVER_URL is not set. Please set it to proceed.") parsed_url = urllib.parse.urlparse(server_url) + + # Set up Ngrok + ngrok_auth_token = os.getenv('NGROK_AUTHTOKEN') + if ngrok_auth_token is not None: + logging.info("Setting up Ngrok") + ngrok_listener = await ngrok.forward(f"{parsed_url.hostname}:{parsed_url.port}", authtoken=ngrok_auth_token) + ngrok_parsed_url = urllib.parse.urlparse(ngrok_listener.url()) + + # Setup SERVER_URL environment variable for device to use + connection_url = f"ws://{ngrok_parsed_url.hostname}" + logging.info(f"Ngrok established at {ngrok_parsed_url.geturl()}") + logging.info(f"\033[1mSERVER_CONNECTION_URL should be set to \"{connection_url}\"\033[0m") + logging.info("Starting `server.py`...") config = Config(app, host=parsed_url.hostname, port=parsed_url.port, lifespan='on') diff --git a/OS/01/start.sh b/OS/01/start.sh index db5689a..71bc436 100755 --- a/OS/01/start.sh +++ b/OS/01/start.sh @@ -4,11 +4,16 @@ # If setting ALL_LOCAL to true, set the path to the WHISPER local model export ALL_LOCAL=False # export WHISPER_MODEL_PATH=... -# export OPENAI_API_KEY=sk-... +export OPENAI_API_KEY="sk-..." + +# Expose through Ngrok +# Uncomment following line with your Ngrok auth token (https://dashboard.ngrok.com/get-started/your-authtoken) +# export NGROK_AUTHTOKEN="AUTH_TOKEN" # If SERVER_START, this is where we'll serve the server. # If DEVICE_START, this is where the device expects the server to be. export SERVER_URL=ws://localhost:8000/ +export SERVER_CONNECTION_URL=$SERVER_URL # Comment if setting up through Ngrok export SERVER_START=True export DEVICE_START=True @@ -24,6 +29,7 @@ export SERVER_EXPOSE_PUBLICALLY=False # export DEBUG_LEVEL=DEBUG export DEBUG_LEVEL="INFO" + ### SETUP # (for dev, reset the ports we were using) @@ -37,6 +43,14 @@ fi start_device() { echo "Starting device..." + if [[ -n $NGROK_AUTHTOKEN ]]; then + echo "Waiting for Ngrok to setup" + sleep 7 + read -p "Enter the Ngrok URL: " ngrok_url + export SERVER_CONNECTION_URL=$ngrok_url + echo "SERVER_CONNECTION_URL set to $SERVER_CONNECTION_URL" + fi + python device.py & DEVICE_PID=$! echo "Device started as process $DEVICE_PID" @@ -64,18 +78,18 @@ stop_processes() { # Trap SIGINT and SIGTERM to stop processes when the script is terminated trap stop_processes SIGINT SIGTERM -# DEVICE -# Start device if DEVICE_START is True -if [[ "$DEVICE_START" == "True" ]]; then - start_device -fi - # SERVER # Start server if SERVER_START is True if [[ "$SERVER_START" == "True" ]]; then start_server fi +# DEVICE +# Start device if DEVICE_START is True +if [[ "$DEVICE_START" == "True" ]]; then + start_device +fi + # Wait for device and server processes to exit wait $DEVICE_PID wait $SERVER_PID From 09194721e82349c6561ec9da0c2e6f95297892f1 Mon Sep 17 00:00:00 2001 From: Zohaib Rauf Date: Sat, 10 Feb 2024 18:58:51 -0800 Subject: [PATCH 2/3] Updated device to remove unecessary waiting --- OS/01/device.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/OS/01/device.py b/OS/01/device.py index c9a8e7d..244ef0d 100644 --- a/OS/01/device.py +++ b/OS/01/device.py @@ -134,6 +134,8 @@ async def message_sender(websocket): async def websocket_communication(WS_URL): while True: try: + logging.info(f"Connecting to `{WS_URL}` ...") + headers = {"ngrok-skip-browser-warning": str(80)} if os.getenv('NGROK_AUTHTOKEN') else {} async with websockets.connect(WS_URL, extra_headers=headers) as websocket: logging.info("Press the spacebar to start/stop recording. Press ESC to exit.") @@ -191,16 +193,6 @@ if __name__ == "__main__": async def main(): WS_URL = os.getenv('SERVER_CONNECTION_URL') - start_time = time.time() - while WS_URL is None: - if time.time() - start_time > 60: - logging.error("SERVER_CONNECTION_URL environment variable is not set after 1 minute. Exiting...") - raise ValueError("The environment variable SERVER_CONNECTION_URL is not set. Please set it to proceed.") - - logging.info("SERVER_CONNECTION_URL environment variable is not set. Waiting...") - await asyncio.sleep(5) # Wait for 5 seconds before checking again - WS_URL = os.getenv('SERVER_CONNECTION_URL') - # Start the WebSocket communication asyncio.create_task(websocket_communication(WS_URL)) From 1666695bf85a524e50105b8e2e9ebb2c3aada350 Mon Sep 17 00:00:00 2001 From: Zohaib Rauf Date: Sat, 10 Feb 2024 20:00:25 -0800 Subject: [PATCH 3/3] Updated --- OS/01/device.py | 5 +++-- OS/01/server.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/OS/01/device.py b/OS/01/device.py index 244ef0d..154cc66 100644 --- a/OS/01/device.py +++ b/OS/01/device.py @@ -13,7 +13,6 @@ import ast from pydub import AudioSegment from pydub.playback import play import io -import time import wave import tempfile from datetime import datetime @@ -136,7 +135,7 @@ async def websocket_communication(WS_URL): try: logging.info(f"Connecting to `{WS_URL}` ...") - headers = {"ngrok-skip-browser-warning": str(80)} if os.getenv('NGROK_AUTHTOKEN') else {} + headers = {"ngrok-skip-browser-warning": str(80), "User-Agent": "project01"} if os.getenv('NGROK_AUTHTOKEN') else {} async with websockets.connect(WS_URL, extra_headers=headers) as websocket: logging.info("Press the spacebar to start/stop recording. Press ESC to exit.") asyncio.create_task(message_sender(websocket)) @@ -192,6 +191,8 @@ if __name__ == "__main__": # Configuration for WebSocket async def main(): WS_URL = os.getenv('SERVER_CONNECTION_URL') + if not WS_URL: + raise ValueError("The environment variable SERVER_URL is not set. Please set it to proceed.") # Start the WebSocket communication asyncio.create_task(websocket_communication(WS_URL)) diff --git a/OS/01/server.py b/OS/01/server.py index 37b56a4..b40c107 100644 --- a/OS/01/server.py +++ b/OS/01/server.py @@ -254,7 +254,7 @@ if __name__ == "__main__": ngrok_parsed_url = urllib.parse.urlparse(ngrok_listener.url()) # Setup SERVER_URL environment variable for device to use - connection_url = f"ws://{ngrok_parsed_url.hostname}" + connection_url = f"wss://{ngrok_parsed_url.hostname}/" logging.info(f"Ngrok established at {ngrok_parsed_url.geturl()}") logging.info(f"\033[1mSERVER_CONNECTION_URL should be set to \"{connection_url}\"\033[0m")