From a6e3105fd77a7de2bab3441cb1f74f70695e942a Mon Sep 17 00:00:00 2001
From: killian <63927363+KillianLucas@users.noreply.github.com>
Date: Sat, 10 Feb 2024 20:54:03 -0800
Subject: [PATCH 1/3] Revert "fix: download whisper local model directly from
huggingface"
---
OS/01/start.sh | 13 +++----------
OS/01/stt.py | 11 +++++------
README.md | 3 ++-
3 files changed, 10 insertions(+), 17 deletions(-)
diff --git a/OS/01/start.sh b/OS/01/start.sh
index 18187bf..8107a17 100755
--- a/OS/01/start.sh
+++ b/OS/01/start.sh
@@ -1,12 +1,10 @@
### SETTINGS
# If ALL_LOCAL is False, we'll use OpenAI's services
-# else we use whisper.cpp and piper local models
+# If setting ALL_LOCAL to true, set the path to the WHISPER local model
export ALL_LOCAL=False
-export WHISPER_MODEL_NAME="ggml-tiny.en.bin"
-
-# Uncomment and set the OpenAI API key for OpenInterpreter to work
-# export OPENAI_API_KEY="sk-..."
+# export WHISPER_MODEL_PATH=...
+export OPENAI_API_KEY="sk-..."
# Expose through Ngrok
# Uncomment following line with your Ngrok auth token (https://dashboard.ngrok.com/get-started/your-authtoken)
@@ -40,11 +38,6 @@ export LOG_LEVEL="INFO"
### SETUP
# if using local models, install the models / executables
-WHISPER_MODEL_URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/"
-WHISPER_RUST_PATH="`pwd`/local_stt/whisper-rust"
-
-curl -OL "${WHISPER_MODEL_URL}${WHISPER_MODEL_NAME}" --output-dir ${WHISPER_RUST_PATH}
-
if [[ "$ALL_LOCAL" == "True" ]]; then
OS=$(uname -s)
ARCH=$(uname -m)
diff --git a/OS/01/stt.py b/OS/01/stt.py
index 48f0ed6..aa14e24 100644
--- a/OS/01/stt.py
+++ b/OS/01/stt.py
@@ -53,14 +53,13 @@ def run_command(command):
return result.stdout, result.stderr
def get_transcription_file(wav_file_path: str):
- whisper_rust_path = os.path.join(os.path.dirname(__file__), 'local_stt', 'whisper-rust')
- model_name = os.getenv('WHISPER_MODEL_NAME')
- if not model_name:
- raise EnvironmentError("WHISPER_MODEL_NAME environment variable is not set.")
+ model_path = os.getenv("WHISPER_MODEL_PATH")
+ if not model_path:
+ raise EnvironmentError("WHISPER_MODEL_PATH environment variable is not set.")
output, error = run_command([
- os.path.join(whisper_rust_path, 'whisper-rust'),
- '--model-path', os.path.join(whisper_rust_path, model_name),
+ os.path.join(os.path.dirname(__file__), 'local_stt', 'whisper-rust', 'whisper-rust'),
+ '--model-path', model_path,
'--file-path', wav_file_path
])
diff --git a/README.md b/README.md
index f092ce9..5350605 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,8 @@ python -m pip install -r requirements.txt
```
NB: Depending on your local Python version, you may run into [this issue↗](https://github.com/TaylorSMarks/playsound/issues/150) installing playsound. Workarounds are provided in the issue.
+If you want to run local speech-to-text from whisper, download the GGML Whisper model from [Huggingface](https://huggingface.co/ggerganov/whisper.cpp). Then in `OS/01/start.sh`, set `ALL_LOCAL=TRUE` and set `WHISPER_MODEL_PATH` to the path of the model.
+
## Usage
```bash
@@ -30,7 +32,6 @@ cd OS/01
bash start.sh
```
-If you want to run local text-to-speech and speech-to-text, set `ALL_LOCAL` in the `start.sh` script to True. This will use the [whisper.cpp](https://github.com/ggerganov/whisper.cpp) and [Piper](https://github.com/rhasspy/piper) models.
## Background
From d275a26afd27cb92b2dec6222340308b966965d0 Mon Sep 17 00:00:00 2001
From: killian <63927363+KillianLucas@users.noreply.github.com>
Date: Sat, 10 Feb 2024 20:55:53 -0800
Subject: [PATCH 2/3] Revert "Added NGrok to expose the server for device to
connect to"
---
OS/01/device.py | 27 ++++++++++++---------------
OS/01/requirements.txt | 26 +++++++++++---------------
OS/01/server.py | 27 ++++++---------------------
OS/01/start.sh | 28 +++++++---------------------
4 files changed, 36 insertions(+), 72 deletions(-)
diff --git a/OS/01/device.py b/OS/01/device.py
index edf7a60..6a1e24f 100644
--- a/OS/01/device.py
+++ b/OS/01/device.py
@@ -6,19 +6,21 @@ from starlette.websockets import WebSocket
from queue import Queue
from pynput import keyboard
import json
+import traceback
import websockets
import queue
+import pydub
import ast
from pydub import AudioSegment
from pydub.playback import play
import io
+import time
import wave
import tempfile
from datetime import datetime
from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run?
from utils.kernel import put_kernel_messages_into_queue
from stt import stt_wav
-import asyncio
from utils.logs import setup_logging
from utils.logs import logger
@@ -32,6 +34,10 @@ RATE = 44100 # Sample rate
RECORDING = False # Flag to control recording state
SPACEBAR_PRESSED = False # Flag to track spacebar press state
+# Configuration for WebSocket
+WS_URL = os.getenv('SERVER_URL')
+if not WS_URL:
+ raise ValueError("The environment variable SERVER_URL is not set. Please set it to proceed.")
# Initialize PyAudio
p = pyaudio.PyAudio()
@@ -121,6 +127,7 @@ def on_release(key):
logger.info("Exiting...")
os._exit(0)
+import asyncio
send_queue = queue.Queue()
@@ -133,12 +140,8 @@ async def message_sender(websocket):
async def websocket_communication(WS_URL):
while True:
try:
- logger.info(f"Connecting to `{WS_URL}` ...")
-
- headers = {"ngrok-skip-browser-warning": str(80), "User-Agent": "project01"} if os.getenv('NGROK_AUTHTOKEN') else {}
- async with websockets.connect(WS_URL, extra_headers=headers) as websocket:
+ async with websockets.connect(WS_URL) as websocket:
logger.info("Press the spacebar to start/stop recording. Press ESC to exit.")
-
asyncio.create_task(message_sender(websocket))
initial_message = {"role": None, "type": None, "format": None, "content": None}
@@ -184,20 +187,14 @@ async def websocket_communication(WS_URL):
result = interpreter.computer.run(language, code)
send_queue.put(result)
-
- except Exception as e:
- logging.exception(f"An error occurred during websocket communication. {e}")
- logging.info(f"Connecting to `{WS_URL}`...")
+ except:
+ # traceback.print_exc()
+ logger.info(f"Connecting to `{WS_URL}`...")
await asyncio.sleep(2)
if __name__ == "__main__":
- # Configuration for WebSocket
async def main():
- WS_URL = os.getenv('SERVER_CONNECTION_URL')
- if not WS_URL:
- raise ValueError("The environment variable SERVER_URL is not set. Please set it to proceed.")
-
# Start the WebSocket communication
asyncio.create_task(websocket_communication(WS_URL))
diff --git a/OS/01/requirements.txt b/OS/01/requirements.txt
index 08d5d35..6e70eb4 100644
--- a/OS/01/requirements.txt
+++ b/OS/01/requirements.txt
@@ -1,16 +1,12 @@
git+https://github.com/KillianLucas/open-interpreter.git
-asyncio==3.4.3
-PyAudio==0.2.14
-pynput==1.7.6
-fastapi==0.109.2
-uvicorn==0.27.1
-websockets==12.0
-playsound==1.3.0
-python-dotenv==1.0.1
-ffmpeg-python==0.2.0
-textual==0.50.1
-pydub==0.25.1
-ngrok==1.0.0
-wheel
-
-
+asyncio
+PyAudio
+pynput
+fastapi
+uvicorn
+websockets
+playsound
+python-dotenv
+ffmpeg-python
+textual
+pydub
\ No newline at end of file
diff --git a/OS/01/server.py b/OS/01/server.py
index 39a67b8..5986c48 100644
--- a/OS/01/server.py
+++ b/OS/01/server.py
@@ -1,12 +1,17 @@
from starlette.websockets import WebSocketDisconnect
import ast
import json
+import time
import queue
import os
import traceback
+from queue import Queue
+from threading import Thread
+import threading
+import uvicorn
import re
from fastapi import FastAPI
-from fastapi.responses import PlainTextResponse
+from threading import Thread
from starlette.websockets import WebSocket
from stt import stt_bytes
from tts import tts
@@ -16,8 +21,6 @@ import urllib.parse
from utils.kernel import put_kernel_messages_into_queue
from i import configure_interpreter
from interpreter import interpreter
-import ngrok
-import signal
from utils.logs import setup_logging
from utils.logs import logger
@@ -87,10 +90,6 @@ if os.getenv('CODE_RUNNER') == "device":
# Configure interpreter
interpreter = configure_interpreter(interpreter)
-@app.get("/ping")
-async def ping():
- return PlainTextResponse("pong")
-
@app.websocket("/")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
@@ -234,7 +233,6 @@ from uvicorn import Config, Server
if __name__ == "__main__":
async def main():
-
# Start listening
asyncio.create_task(listener())
@@ -246,19 +244,6 @@ if __name__ == "__main__":
if not server_url:
raise ValueError("The environment variable SERVER_URL is not set. Please set it to proceed.")
parsed_url = urllib.parse.urlparse(server_url)
-
- # Set up Ngrok
- ngrok_auth_token = os.getenv('NGROK_AUTHTOKEN')
- if ngrok_auth_token is not None:
- logger.info("Setting up Ngrok")
- ngrok_listener = await ngrok.forward(f"{parsed_url.hostname}:{parsed_url.port}", authtoken=ngrok_auth_token)
- ngrok_parsed_url = urllib.parse.urlparse(ngrok_listener.url())
-
- # Setup SERVER_URL environment variable for device to use
- connection_url = f"wss://{ngrok_parsed_url.hostname}/"
- logger.info(f"Ngrok established at {ngrok_parsed_url.geturl()}")
- logger.info(f"\033[1mSERVER_CONNECTION_URL should be set to \"{connection_url}\"\033[0m")
-
logger.info("Starting `server.py`...")
config = Config(app, host=parsed_url.hostname, port=parsed_url.port, lifespan='on')
diff --git a/OS/01/start.sh b/OS/01/start.sh
index 8107a17..8a4dcc6 100755
--- a/OS/01/start.sh
+++ b/OS/01/start.sh
@@ -4,11 +4,7 @@
# If setting ALL_LOCAL to true, set the path to the WHISPER local model
export ALL_LOCAL=False
# export WHISPER_MODEL_PATH=...
-export OPENAI_API_KEY="sk-..."
-
-# Expose through Ngrok
-# Uncomment following line with your Ngrok auth token (https://dashboard.ngrok.com/get-started/your-authtoken)
-# export NGROK_AUTHTOKEN="AUTH_TOKEN"
+# export OPENAI_API_KEY=sk-...
# For TTS, we use the en_US-lessac-medium voice model by default
# Please change the voice URL and voice name if you wish to use another voice
@@ -18,7 +14,6 @@ export PIPER_VOICE_NAME="en_US-lessac-medium.onnx"
# If SERVER_START, this is where we'll serve the server.
# If DEVICE_START, this is where the device expects the server to be.
export SERVER_URL=ws://localhost:8000/
-export SERVER_CONNECTION_URL=$SERVER_URL # Comment if setting up through Ngrok
export SERVER_START=True
export DEVICE_START=True
@@ -34,7 +29,6 @@ export SERVER_EXPOSE_PUBLICALLY=False
# export LOG_LEVEL=DEBUG
export LOG_LEVEL="INFO"
-
### SETUP
# if using local models, install the models / executables
@@ -88,14 +82,6 @@ fi
start_device() {
echo "Starting device..."
- if [[ -n $NGROK_AUTHTOKEN ]]; then
- echo "Waiting for Ngrok to setup"
- sleep 7
- read -p "Enter the Ngrok URL: " ngrok_url
- export SERVER_CONNECTION_URL=$ngrok_url
- echo "SERVER_CONNECTION_URL set to $SERVER_CONNECTION_URL"
- fi
-
python device.py &
DEVICE_PID=$!
echo "Device started as process $DEVICE_PID"
@@ -123,18 +109,18 @@ stop_processes() {
# Trap SIGINT and SIGTERM to stop processes when the script is terminated
trap stop_processes SIGINT SIGTERM
-# SERVER
-# Start server if SERVER_START is True
-if [[ "$SERVER_START" == "True" ]]; then
- start_server
-fi
-
# DEVICE
# Start device if DEVICE_START is True
if [[ "$DEVICE_START" == "True" ]]; then
start_device
fi
+# SERVER
+# Start server if SERVER_START is True
+if [[ "$SERVER_START" == "True" ]]; then
+ start_server
+fi
+
# Wait for device and server processes to exit
wait $DEVICE_PID
wait $SERVER_PID
From 35ea81c61dd730ba9b708011690516f880d86e41 Mon Sep 17 00:00:00 2001
From: Tom Chapin
Date: Sat, 10 Feb 2024 21:22:10 -0800
Subject: [PATCH 3/3] Re-applying shivan's whisper changes after the revert
---
OS/01/start.sh | 11 +++++++++--
OS/01/stt.py | 11 ++++++-----
README.md | 3 +--
3 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/OS/01/start.sh b/OS/01/start.sh
index 8a4dcc6..0ff76f0 100755
--- a/OS/01/start.sh
+++ b/OS/01/start.sh
@@ -1,9 +1,11 @@
### SETTINGS
# If ALL_LOCAL is False, we'll use OpenAI's services
-# If setting ALL_LOCAL to true, set the path to the WHISPER local model
+# else we use whisper.cpp and piper local models
export ALL_LOCAL=False
-# export WHISPER_MODEL_PATH=...
+export WHISPER_MODEL_NAME="ggml-tiny.en.bin"
+
+# Set the OpenAI API key for OpenInterpreter to work
# export OPENAI_API_KEY=sk-...
# For TTS, we use the en_US-lessac-medium voice model by default
@@ -32,6 +34,11 @@ export LOG_LEVEL="INFO"
### SETUP
# if using local models, install the models / executables
+WHISPER_MODEL_URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/"
+WHISPER_RUST_PATH="`pwd`/local_stt/whisper-rust"
+
+curl -OL "${WHISPER_MODEL_URL}${WHISPER_MODEL_NAME}" --output-dir ${WHISPER_RUST_PATH}
+
if [[ "$ALL_LOCAL" == "True" ]]; then
OS=$(uname -s)
ARCH=$(uname -m)
diff --git a/OS/01/stt.py b/OS/01/stt.py
index aa14e24..48f0ed6 100644
--- a/OS/01/stt.py
+++ b/OS/01/stt.py
@@ -53,13 +53,14 @@ def run_command(command):
return result.stdout, result.stderr
def get_transcription_file(wav_file_path: str):
- model_path = os.getenv("WHISPER_MODEL_PATH")
- if not model_path:
- raise EnvironmentError("WHISPER_MODEL_PATH environment variable is not set.")
+ whisper_rust_path = os.path.join(os.path.dirname(__file__), 'local_stt', 'whisper-rust')
+ model_name = os.getenv('WHISPER_MODEL_NAME')
+ if not model_name:
+ raise EnvironmentError("WHISPER_MODEL_NAME environment variable is not set.")
output, error = run_command([
- os.path.join(os.path.dirname(__file__), 'local_stt', 'whisper-rust', 'whisper-rust'),
- '--model-path', model_path,
+ os.path.join(whisper_rust_path, 'whisper-rust'),
+ '--model-path', os.path.join(whisper_rust_path, model_name),
'--file-path', wav_file_path
])
diff --git a/README.md b/README.md
index 5350605..f092ce9 100644
--- a/README.md
+++ b/README.md
@@ -23,8 +23,6 @@ python -m pip install -r requirements.txt
```
NB: Depending on your local Python version, you may run into [this issue↗](https://github.com/TaylorSMarks/playsound/issues/150) installing playsound. Workarounds are provided in the issue.
-If you want to run local speech-to-text from whisper, download the GGML Whisper model from [Huggingface](https://huggingface.co/ggerganov/whisper.cpp). Then in `OS/01/start.sh`, set `ALL_LOCAL=TRUE` and set `WHISPER_MODEL_PATH` to the path of the model.
-
## Usage
```bash
@@ -32,6 +30,7 @@ cd OS/01
bash start.sh
```
+If you want to run local text-to-speech and speech-to-text, set `ALL_LOCAL` in the `start.sh` script to True. This will use the [whisper.cpp](https://github.com/ggerganov/whisper.cpp) and [Piper](https://github.com/rhasspy/piper) models.
## Background