From f25586500045aaf72adbfd2a1d82ddbeee05d872 Mon Sep 17 00:00:00 2001
From: killian <63927363+KillianLucas@users.noreply.github.com>
Date: Sat, 3 Feb 2024 19:01:11 -0800
Subject: [PATCH] a new start

---
 OS/01/assistant/assistant.py | 32 +++++++++++++++++++++++++++++---
 OS/01/start.sh               | 28 ++++++++++------------------
 2 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/OS/01/assistant/assistant.py b/OS/01/assistant/assistant.py
index f424a22..ca9c73a 100644
--- a/OS/01/assistant/assistant.py
+++ b/OS/01/assistant/assistant.py
@@ -12,10 +12,13 @@ import queue
 import os
 from threading import Thread
 import uvicorn
+import re
 from fastapi import FastAPI
 from threading import Thread
 from starlette.websockets import WebSocket
 from create_interpreter import create_interpreter
+from stt import stt
+from tts import tts
 
 # Create interpreter
 interpreter = create_interpreter()
@@ -27,12 +30,18 @@ conversation_history_path = os.path.join(script_dir, 'conversations', 'user.json
 to_user = queue.Queue()
 to_assistant = queue.Queue()
 
+# This is so we only say() full sentences
+accumulated_text = ""
+def is_full_sentence(text):
+    return text.endswith(('.', '!', '?'))
+def split_into_sentences(text):
+    return re.split(r'(?<=[.!?])\s+', text)
+
 app = FastAPI()
 
 @app.post("/computer")
 async def read_computer(item: dict):
     to_assistant.put(item)
-    return {"message": "Item added to queue"}
 
 @app.websocket("/user")
 async def websocket_endpoint(websocket: WebSocket):
@@ -74,13 +83,30 @@ def queue_listener():
         messages.append(message)
         with open(conversation_history_path, 'w') as file:
             json.dump(messages, file)
+
+        accumulated_text = ""
         
         for chunk in interpreter.chat(messages):
 
-            # Send it to the interface
+            # Send it to the user
             to_user.put(chunk)
+            
+            # Speak full sentences out loud
+            accumulated_text += chunk["content"]
+            sentences = split_into_sentences(accumulated_text)
+            if is_full_sentence(sentences[-1]):
+                for sentence in sentences:
+                    for audio_chunk in tts(sentence):
+                        to_user.put(audio_chunk)
+                accumulated_text = ""
+            else:
+                for sentence in sentences[:-1]:
+                    for audio_chunk in tts(sentence):
+                        to_user.put(audio_chunk)
+                accumulated_text = sentences[-1]
 
-            # Stream audio chunks
+            if chunk["type"] == "message" and "content" in sentence:
+                sentence += chunk.get("content")
             
             # If we have a new message, save our progress and go back to the top
             if not to_assistant.empty():
diff --git a/OS/01/start.sh b/OS/01/start.sh
index 9689c4b..41c5ce3 100755
--- a/OS/01/start.sh
+++ b/OS/01/start.sh
@@ -6,17 +6,15 @@ sudo apt-get update
 sudo apt-get install redis-server
 pip install -r requirements.txt
 
-# START REDIS
-
-redis-cli -h localhost -p 6379 rpush to_interface ""
-redis-cli -h localhost -p 6379 rpush to_core ""
+### COMPUTER
 
+# START KERNEL WATCHER
 
-### CORE
+python computer/kernel_watcher.py &
 
-# START KERNEL WATCHER
+# START RUN ENDPOINT
 
-python core/kernel_watcher.py &
+python computer/run.py
 
 # START SST AND TTS SERVICES
 
@@ -28,18 +26,12 @@ python core/kernel_watcher.py &
 # (disabled, we'll start with hosted services)
 # python core/llm/start.py &
 
-# START CORE
-
-python core/start_core.py &
-
-
-### INTERFACE
+# START ASSISTANT
 
-# START INTERFACE
+python assistant/assistant.py &
 
-python interface/interface.py &
+### USER
 
-# START DISPLAY
+# START USER
 
-# (this should be changed to run it in fullscreen / kiosk mode)
-open interface/display.html
\ No newline at end of file
+python user/user.py &
\ No newline at end of file