listen > assistant

OS/01/assistant/assistant.py

@@ -53,7 +53,7 @@ async def websocket_endpoint(websocket: WebSocket):
             message = to_user.get()
             await websocket.send_json(message)
 
-audio_chunks = []
+audio_file = bytearray()
 
 def queue_listener():
     while True:
@@ -65,11 +65,11 @@ def queue_listener():
         # Hold the audio in a buffer. If it's ready (we got end flag, stt it)
         if message["type"] == "audio":
             if "content" in message:
-                audio_chunks.append(message)
+                audio_file.extend(message["content"])
             if "end" in message:
-                text = stt(audio_chunks)
-                audio_chunks = []
-                message = {"role": "user", "type": "message", "content": text}
+                content = stt(audio_file, message["format"])
+                audio_file = bytearray()
+                message = {"role": "user", "type": "message", "content": content}
             else:
                 continue
 

OS/01/user/record.py

@@ -52,7 +52,7 @@ async def start_recording():
     print("Recording started...")
     async with websockets.connect("ws://localhost:8000/user") as websocket:
         # Send the start command with mime type
-        await websocket.send(json.dumps({"action": "command", "state": "start", "mimeType": "audio/wav"}))
+        await websocket.send(json.dumps({"role": "user", "type": "audio", "format": "audio/wav", "start": True}))
         while recording:
             data = stream.read(chunk)
             frames.append(data)
@@ -65,13 +65,13 @@ async def start_recording():
             with open(file_path, 'rb') as audio_file:
                 byte_chunk = audio_file.read(ws_chunk_size)
                 while byte_chunk:
-                    await websocket.send(byte_chunk)
+                    await websocket.send({"role": "user", "type": "audio", "format": "audio/wav", "content": byte_chunk})
                     byte_chunk = audio_file.read(ws_chunk_size)
         finally:
             os.remove(file_path)
 
         # Send the end command
-        await websocket.send(json.dumps({"action": "command", "state": "end"}))
+        await websocket.send(json.dumps({"role": "user", "type": "audio", "format": "audio/wav", "end": True}))
 
         # Receive a json message and then close the connection
         message = await websocket.recv()