|
|
|
|
@ -24,6 +24,7 @@ import tempfile
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
import cv2
|
|
|
|
|
import base64
|
|
|
|
|
import platform
|
|
|
|
|
from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run?
|
|
|
|
|
# In the future, I guess kernel watching code should be elsewhere? Somewhere server / client agnostic?
|
|
|
|
|
from ..server.utils.kernel import put_kernel_messages_into_queue
|
|
|
|
|
@ -58,6 +59,7 @@ CAMERA_WARMUP_SECONDS = float(os.getenv('CAMERA_WARMUP_SECONDS', 0))
|
|
|
|
|
|
|
|
|
|
# Specify OS
|
|
|
|
|
current_platform = get_system_info()
|
|
|
|
|
is_win10 = lambda: platform.system() == "Windows" and "10" in platform.version()
|
|
|
|
|
|
|
|
|
|
# Initialize PyAudio
|
|
|
|
|
p = pyaudio.PyAudio()
|
|
|
|
|
@ -252,65 +254,79 @@ class Device:
|
|
|
|
|
|
|
|
|
|
async def websocket_communication(self, WS_URL):
|
|
|
|
|
show_connection_log = True
|
|
|
|
|
while True:
|
|
|
|
|
try:
|
|
|
|
|
async with websockets.connect(WS_URL) as websocket:
|
|
|
|
|
if CAMERA_ENABLED:
|
|
|
|
|
print("\nHold the spacebar to start recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.")
|
|
|
|
|
else:
|
|
|
|
|
print("\nHold the spacebar to start recording. Press CTRL-C to exit.")
|
|
|
|
|
|
|
|
|
|
asyncio.create_task(self.message_sender(websocket))
|
|
|
|
|
async def exec_ws_communication(websocket):
|
|
|
|
|
if CAMERA_ENABLED:
|
|
|
|
|
print("\nHold the spacebar to start recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.")
|
|
|
|
|
else:
|
|
|
|
|
print("\nHold the spacebar to start recording. Press CTRL-C to exit.")
|
|
|
|
|
|
|
|
|
|
while True:
|
|
|
|
|
await asyncio.sleep(0.01)
|
|
|
|
|
chunk = await websocket.recv()
|
|
|
|
|
asyncio.create_task(self.message_sender(websocket))
|
|
|
|
|
|
|
|
|
|
logger.debug(f"Got this message from the server: {type(chunk)} {chunk}")
|
|
|
|
|
while True:
|
|
|
|
|
await asyncio.sleep(0.01)
|
|
|
|
|
chunk = await websocket.recv()
|
|
|
|
|
|
|
|
|
|
if type(chunk) == str:
|
|
|
|
|
chunk = json.loads(chunk)
|
|
|
|
|
logger.debug(f"Got this message from the server: {type(chunk)} {chunk}")
|
|
|
|
|
|
|
|
|
|
message = accumulator.accumulate(chunk)
|
|
|
|
|
if message == None:
|
|
|
|
|
# Will be None until we have a full message ready
|
|
|
|
|
continue
|
|
|
|
|
if type(chunk) == str:
|
|
|
|
|
chunk = json.loads(chunk)
|
|
|
|
|
|
|
|
|
|
# At this point, we have our message
|
|
|
|
|
message = accumulator.accumulate(chunk)
|
|
|
|
|
if message == None:
|
|
|
|
|
# Will be None until we have a full message ready
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
if message["type"] == "audio" and message["format"].startswith("bytes"):
|
|
|
|
|
# At this point, we have our message
|
|
|
|
|
|
|
|
|
|
# Convert bytes to audio file
|
|
|
|
|
if message["type"] == "audio" and message["format"].startswith("bytes"):
|
|
|
|
|
|
|
|
|
|
audio_bytes = message["content"]
|
|
|
|
|
# Convert bytes to audio file
|
|
|
|
|
|
|
|
|
|
# Create an AudioSegment instance with the raw data
|
|
|
|
|
audio = AudioSegment(
|
|
|
|
|
# raw audio data (bytes)
|
|
|
|
|
data=audio_bytes,
|
|
|
|
|
# signed 16-bit little-endian format
|
|
|
|
|
sample_width=2,
|
|
|
|
|
# 16,000 Hz frame rate
|
|
|
|
|
frame_rate=16000,
|
|
|
|
|
# mono sound
|
|
|
|
|
channels=1
|
|
|
|
|
)
|
|
|
|
|
audio_bytes = message["content"]
|
|
|
|
|
|
|
|
|
|
self.audiosegments.append(audio)
|
|
|
|
|
# Create an AudioSegment instance with the raw data
|
|
|
|
|
audio = AudioSegment(
|
|
|
|
|
# raw audio data (bytes)
|
|
|
|
|
data=audio_bytes,
|
|
|
|
|
# signed 16-bit little-endian format
|
|
|
|
|
sample_width=2,
|
|
|
|
|
# 16,000 Hz frame rate
|
|
|
|
|
frame_rate=16000,
|
|
|
|
|
# mono sound
|
|
|
|
|
channels=1
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Run the code if that's the client's job
|
|
|
|
|
if os.getenv('CODE_RUNNER') == "client":
|
|
|
|
|
if message["type"] == "code" and "end" in message:
|
|
|
|
|
language = message["format"]
|
|
|
|
|
code = message["content"]
|
|
|
|
|
result = interpreter.computer.run(language, code)
|
|
|
|
|
send_queue.put(result)
|
|
|
|
|
except:
|
|
|
|
|
logger.debug(traceback.format_exc())
|
|
|
|
|
if show_connection_log:
|
|
|
|
|
logger.info(f"Connecting to `{WS_URL}`...")
|
|
|
|
|
show_connection_log = False
|
|
|
|
|
await asyncio.sleep(2)
|
|
|
|
|
self.audiosegments.append(audio)
|
|
|
|
|
|
|
|
|
|
# Run the code if that's the client's job
|
|
|
|
|
if os.getenv('CODE_RUNNER') == "client":
|
|
|
|
|
if message["type"] == "code" and "end" in message:
|
|
|
|
|
language = message["format"]
|
|
|
|
|
code = message["content"]
|
|
|
|
|
result = interpreter.computer.run(language, code)
|
|
|
|
|
send_queue.put(result)
|
|
|
|
|
|
|
|
|
|
if is_win10():
|
|
|
|
|
logger.info('Windows 10 detected')
|
|
|
|
|
# Workaround for Windows 10 not latching to the websocket server.
|
|
|
|
|
# See https://github.com/OpenInterpreter/01/issues/197
|
|
|
|
|
try:
|
|
|
|
|
ws = websockets.connect(WS_URL)
|
|
|
|
|
await exec_ws_communication(ws)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
logger.error(f"Error while attempting to connect: {e}")
|
|
|
|
|
else:
|
|
|
|
|
while True:
|
|
|
|
|
try:
|
|
|
|
|
async with websockets.connect(WS_URL) as websocket:
|
|
|
|
|
await exec_ws_communication(websocket)
|
|
|
|
|
except:
|
|
|
|
|
logger.debug(traceback.format_exc())
|
|
|
|
|
if show_connection_log:
|
|
|
|
|
logger.info(f"Connecting to `{WS_URL}`...")
|
|
|
|
|
show_connection_log = False
|
|
|
|
|
await asyncio.sleep(2)
|
|
|
|
|
|
|
|
|
|
async def start_async(self):
|
|
|
|
|
# Configuration for WebSocket
|
|
|
|
|
|
Ty Fiero
10 months ago
committed by
GitHub