Ben Xu
7 months ago
parent
c35d4c08f4
commit
10681b552f
File diff suppressed because one or more lines are too long
@ -0,0 +1,119 @@
|
||||
import asyncio
|
||||
import traceback
|
||||
import json
|
||||
from fastapi import FastAPI, WebSocket, Header
|
||||
from uvicorn import Config, Server
|
||||
from interpreter import interpreter as base_interpreter
|
||||
from .async_interpreter import AsyncInterpreter
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from typing import List, Dict, Any
|
||||
from openai import OpenAI
|
||||
from pydantic import BaseModel
|
||||
import argparse
|
||||
import os
|
||||
|
||||
os.environ["STT_RUNNER"] = "server"
|
||||
os.environ["TTS_RUNNER"] = "server"
|
||||
|
||||
# Parse command line arguments for port number
|
||||
parser = argparse.ArgumentParser(description="FastAPI server.")
|
||||
parser.add_argument("--port", type=int, default=8000, help="Port to run on.")
|
||||
args = parser.parse_args()
|
||||
|
||||
base_interpreter.tts = "openai"
|
||||
base_interpreter.llm.model = "gpt-4-turbo"
|
||||
|
||||
|
||||
async def main():
|
||||
interpreter = AsyncInterpreter(base_interpreter)
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"], # Allow all methods (GET, POST, etc.)
|
||||
allow_headers=["*"], # Allow all headers
|
||||
)
|
||||
|
||||
@app.post("/load_chat")
|
||||
async def load_chat(messages: List[Dict[str, Any]]):
|
||||
interpreter.interpreter.messages = messages
|
||||
interpreter.active_chat_messages = messages
|
||||
print("🪼🪼🪼🪼🪼🪼 Messages loaded: ", interpreter.active_chat_messages)
|
||||
return {"status": "success"}
|
||||
|
||||
@app.websocket("/ws")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
try:
|
||||
|
||||
async def receive_input():
|
||||
while True:
|
||||
data = await websocket.receive()
|
||||
|
||||
if isinstance(data, bytes):
|
||||
await interpreter.input(data)
|
||||
elif "bytes" in data:
|
||||
await interpreter.input(data["bytes"])
|
||||
print("SERVER FEEDING AUDIO")
|
||||
elif "text" in data:
|
||||
print("RECEIVED INPUT", data)
|
||||
await interpreter.input(data["text"])
|
||||
|
||||
async def send_output():
|
||||
while True:
|
||||
output = await interpreter.output()
|
||||
if isinstance(output, bytes):
|
||||
await websocket.send_bytes(output)
|
||||
# we dont send out bytes rn, no TTS
|
||||
pass
|
||||
elif isinstance(output, dict):
|
||||
await websocket.send_text(json.dumps(output))
|
||||
|
||||
await asyncio.gather(receive_input(), send_output())
|
||||
except Exception as e:
|
||||
print(f"WebSocket connection closed with exception: {e}")
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
await websocket.close()
|
||||
|
||||
config = Config(app, host="0.0.0.0", port=8000, lifespan="on")
|
||||
server = Server(config)
|
||||
await server.serve()
|
||||
|
||||
class Rename(BaseModel):
|
||||
input: str
|
||||
|
||||
@app.post("/rename-chat")
|
||||
async def rename_chat(body_content: Rename, x_api_key: str = Header(None)):
|
||||
print("RENAME CHAT REQUEST in PY 🌙🌙🌙🌙")
|
||||
input_value = body_content.input
|
||||
client = OpenAI(
|
||||
# defaults to os.environ.get("OPENAI_API_KEY")
|
||||
api_key=x_api_key,
|
||||
)
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"Given the following chat snippet, create a unique and descriptive title in less than 8 words. Your answer must not be related to customer service.\n\n{input_value}",
|
||||
}
|
||||
],
|
||||
temperature=0.3,
|
||||
stream=False,
|
||||
)
|
||||
print(response)
|
||||
completion = response["choices"][0]["message"]["content"]
|
||||
return {"data": {"content": completion}}
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
traceback.print_exc()
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@ -0,0 +1,211 @@
|
||||
# This is a websocket interpreter, TTS and STT disabled.
|
||||
# It makes a websocket on port 8000 that sends/recieves LMC messages in *streaming* format.
|
||||
|
||||
### You MUST send a start and end flag with each message! For example: ###
|
||||
|
||||
"""
|
||||
{"role": "user", "type": "message", "start": True})
|
||||
{"role": "user", "type": "message", "content": "hi"})
|
||||
{"role": "user", "type": "message", "end": True})
|
||||
"""
|
||||
|
||||
###
|
||||
|
||||
from pynput import keyboard
|
||||
from RealtimeTTS import TextToAudioStream, OpenAIEngine, CoquiEngine
|
||||
from RealtimeSTT import AudioToTextRecorder
|
||||
import time
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
|
||||
class AsyncInterpreter:
|
||||
def __init__(self, interpreter):
|
||||
self.interpreter = interpreter
|
||||
|
||||
# STT
|
||||
self.stt = AudioToTextRecorder(use_microphone=False)
|
||||
self.stt.stop() # It needs this for some reason
|
||||
|
||||
# TTS
|
||||
if self.interpreter.tts == "coqui":
|
||||
engine = CoquiEngine()
|
||||
elif self.interpreter.tts == "openai":
|
||||
engine = OpenAIEngine()
|
||||
self.tts = TextToAudioStream(engine)
|
||||
|
||||
self.active_chat_messages = []
|
||||
|
||||
self._input_queue = asyncio.Queue() # Queue that .input will shove things into
|
||||
self._output_queue = asyncio.Queue() # Queue to put output chunks into
|
||||
self._last_lmc_start_flag = None # Unix time of last LMC start flag recieved
|
||||
self._in_keyboard_write_block = (
|
||||
False # Tracks whether interpreter is trying to use the keyboard
|
||||
)
|
||||
self.loop = asyncio.get_event_loop()
|
||||
|
||||
async def _add_to_queue(self, queue, item):
|
||||
await queue.put(item)
|
||||
|
||||
async def clear_queue(self, queue):
|
||||
while not queue.empty():
|
||||
await queue.get()
|
||||
|
||||
async def clear_input_queue(self):
|
||||
await self.clear_queue(self._input_queue)
|
||||
|
||||
async def clear_output_queue(self):
|
||||
await self.clear_queue(self._output_queue)
|
||||
|
||||
async def input(self, chunk):
|
||||
"""
|
||||
Expects a chunk in streaming LMC format.
|
||||
"""
|
||||
if isinstance(chunk, bytes):
|
||||
# It's probably a chunk of audio
|
||||
self.stt.feed_audio(chunk)
|
||||
print("INTERPRETER FEEDING AUDIO")
|
||||
|
||||
else:
|
||||
|
||||
try:
|
||||
chunk = json.loads(chunk)
|
||||
except:
|
||||
pass
|
||||
|
||||
if "start" in chunk:
|
||||
print("input received")
|
||||
self.stt.start()
|
||||
self._last_lmc_start_flag = time.time()
|
||||
# self.interpreter.computer.terminal.stop() # Stop any code execution... maybe we should make interpreter.stop()?
|
||||
elif "end" in chunk:
|
||||
print("running oi on input now")
|
||||
asyncio.create_task(self.run())
|
||||
else:
|
||||
await self._add_to_queue(self._input_queue, chunk)
|
||||
|
||||
def add_to_output_queue_sync(self, chunk):
|
||||
"""
|
||||
Synchronous function to add a chunk to the output queue.
|
||||
"""
|
||||
print("ADDING TO QUEUE:", chunk)
|
||||
asyncio.create_task(self._add_to_queue(self._output_queue, chunk))
|
||||
|
||||
async def run(self):
|
||||
"""
|
||||
Runs OI on the audio bytes submitted to the input. Will add streaming LMC chunks to the _output_queue.
|
||||
"""
|
||||
self.interpreter.messages = self.active_chat_messages
|
||||
|
||||
# self.beeper.start()
|
||||
|
||||
self.stt.stop()
|
||||
# message = self.stt.text()
|
||||
# print("THE MESSAGE:", message)
|
||||
|
||||
# accumulates the input queue message
|
||||
input_queue = []
|
||||
while not self._input_queue.empty():
|
||||
input_queue.append(self._input_queue.get())
|
||||
|
||||
print("INPUT QUEUE:", input_queue)
|
||||
# message = [i for i in input_queue if i["type"] == "message"][0]["content"]
|
||||
# message = self.stt.text()
|
||||
|
||||
message = "hello"
|
||||
print(message)
|
||||
|
||||
# print(message)
|
||||
def generate(message):
|
||||
last_lmc_start_flag = self._last_lmc_start_flag
|
||||
self.interpreter.messages = self.active_chat_messages
|
||||
print(
|
||||
"🍀🍀🍀🍀GENERATING, using these messages: ", self.interpreter.messages
|
||||
)
|
||||
print("🍀 🍀 🍀 🍀 active_chat_messages: ", self.active_chat_messages)
|
||||
print("message is", message)
|
||||
|
||||
for chunk in self.interpreter.chat(message, display=True, stream=True):
|
||||
|
||||
if self._last_lmc_start_flag != last_lmc_start_flag:
|
||||
# self.beeper.stop()
|
||||
break
|
||||
|
||||
# self.add_to_output_queue_sync(chunk) # To send text, not just audio
|
||||
|
||||
content = chunk.get("content")
|
||||
|
||||
# Handle message blocks
|
||||
if chunk.get("type") == "message":
|
||||
if content:
|
||||
# self.beeper.stop()
|
||||
|
||||
# Experimental: The AI voice sounds better with replacements like these, but it should happen at the TTS layer
|
||||
# content = content.replace(". ", ". ... ").replace(", ", ", ... ").replace("!", "! ... ").replace("?", "? ... ")
|
||||
|
||||
yield content
|
||||
|
||||
# Handle code blocks
|
||||
elif chunk.get("type") == "code":
|
||||
if "start" in chunk:
|
||||
# self.beeper.start()
|
||||
pass
|
||||
|
||||
# Experimental: If the AI wants to type, we should type immediatly
|
||||
if (
|
||||
self.interpreter.messages[-1]
|
||||
.get("content", "")
|
||||
.startswith("computer.keyboard.write(")
|
||||
):
|
||||
keyboard.controller.type(content)
|
||||
self._in_keyboard_write_block = True
|
||||
if "end" in chunk and self._in_keyboard_write_block:
|
||||
self._in_keyboard_write_block = False
|
||||
# (This will make it so it doesn't type twice when the block executes)
|
||||
if self.interpreter.messages[-1]["content"].startswith(
|
||||
"computer.keyboard.write("
|
||||
):
|
||||
self.interpreter.messages[-1]["content"] = (
|
||||
"dummy_variable = ("
|
||||
+ self.interpreter.messages[-1]["content"][
|
||||
len("computer.keyboard.write(") :
|
||||
]
|
||||
)
|
||||
|
||||
# Send a completion signal
|
||||
|
||||
# self.add_to_output_queue_sync({"role": "server","type": "completion", "content": "DONE"})
|
||||
|
||||
# Feed generate to RealtimeTTS
|
||||
self.add_to_output_queue_sync(
|
||||
{"role": "assistant", "type": "audio", "format": "bytes.wav", "start": True}
|
||||
)
|
||||
self.tts.feed(generate(message))
|
||||
self.tts.play_async(on_audio_chunk=self.on_tts_chunk, muted=True)
|
||||
while True:
|
||||
if self.tts.is_playing():
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
while True:
|
||||
await asyncio.sleep(0.1)
|
||||
print("is_playing", self.tts.is_playing())
|
||||
if not self.tts.is_playing():
|
||||
self.add_to_output_queue_sync(
|
||||
{
|
||||
"role": "assistant",
|
||||
"type": "audio",
|
||||
"format": "bytes.wav",
|
||||
"end": True,
|
||||
}
|
||||
)
|
||||
break
|
||||
|
||||
async def _on_tts_chunk_async(self, chunk):
|
||||
print("SENDING TTS CHUNK")
|
||||
await self._add_to_queue(self._output_queue, chunk)
|
||||
|
||||
def on_tts_chunk(self, chunk):
|
||||
asyncio.run(self._on_tts_chunk_async(chunk))
|
||||
|
||||
async def output(self):
|
||||
return await self._output_queue.get()
|
||||
Loading…
Reference in new issue