open-interpreter-ai
/
01
mirror of https://github.com/OpenInterpreter/01
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix missing Device for Windows

Browse Source
pull/192/head
Davy Peter Braun 1 year ago
parent 7d1a4d9777
commit 621c5cd048
6 changed files with 212 additions and 169 deletions
Show Stats Download Patch File Download Diff File

111
software/source/clients/base_device.py
Unescape View File

@ -1,5 +1,8 @@
from dotenv import load_dotenv from dotenv import load_dotenv
from source.server.utils.logs import setup_logging, logger
load_dotenv() # take environment variables from .env. load_dotenv() # take environment variables from .env.
setup_logging()
import os import os
import asyncio import asyncio
@ -11,7 +14,7 @@ from queue import Queue
from pynput import keyboard from pynput import keyboard
import json import json
import traceback import traceback
import websockets import websocket as wsc
import queue import queue
import pydub import pydub
import ast import ast
@ -26,7 +29,7 @@ import cv2
import base64 import base64
from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run? from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run?
# In the future, I guess kernel watching code should be elsewhere? Somewhere server / client agnostic? # In the future, I guess kernel watching code should be elsewhere? Somewhere server / client agnostic?
from ..server.utils.kernel import put_kernel_messages_into_queue from ..server.utils.kernel import KernelChecker, put_kernel_messages_into_queue
from ..server.utils.get_system_info import get_system_info from ..server.utils.get_system_info import get_system_info
from ..server.utils.process_utils import kill_process_tree from ..server.utils.process_utils import kill_process_tree
@ -41,13 +44,11 @@ from ..utils.accumulator import Accumulator
accumulator = Accumulator() accumulator = Accumulator()
# Configuration for Audio Recording # AudioSegment configuration
CHUNK = 1024 # Record in chunks of 1024 samples AUDIO_SAMPLE_WIDTH = 2
FORMAT = pyaudio.paInt16 # 16 bits per sample AUDIO_FRAME_RATE = 16000
CHANNELS = 1 # Mono AUDIO_MONO_CHANNEL = 1
RATE = 44100 # Sample rate AUDIO_CODE_RUNNER_CLIENT = "client"
RECORDING = False # Flag to control recording state
SPACEBAR_PRESSED = False # Flag to track spacebar press state
# Camera configuration # Camera configuration
CAMERA_ENABLED = os.getenv('CAMERA_ENABLED', False) CAMERA_ENABLED = os.getenv('CAMERA_ENABLED', False)
@ -71,6 +72,14 @@ class Device:
self.audiosegments = [] self.audiosegments = []
self.server_url = "" self.server_url = ""
# Configuration for Audio Recording
self.CHUNK = 1024 # Record in chunks of 1024 samples
self.FORMAT = pyaudio.paInt16 # 16 bits per sample
self.CHANNELS = 1 # Mono
self.RATE = 44100 # Sample rate
self.RECORDING = False # Flag to control recording state
self.SPACEBAR_PRESSED = False # Flag to track spacebar press state
def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX): def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX):
"""Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list.""" """Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list."""
image_path = None image_path = None
@ -141,7 +150,6 @@ class Device:
def record_audio(self): def record_audio(self):
if os.getenv('STT_RUNNER') == "server": if os.getenv('STT_RUNNER') == "server":
# STT will happen on the server. we're sending audio. # STT will happen on the server. we're sending audio.
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "start": True}) send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "start": True})
@ -152,20 +160,19 @@ class Device:
raise Exception("STT_RUNNER must be set to either 'client' or 'server'.") raise Exception("STT_RUNNER must be set to either 'client' or 'server'.")
"""Record audio from the microphone and add it to the queue.""" """Record audio from the microphone and add it to the queue."""
stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=CHUNK) stream = p.open(format=self.FORMAT, channels=self.CHANNELS, rate=self.RATE, input=True, frames_per_buffer=self.CHUNK)
print("Recording started...") print("Recording started...")
global RECORDING
# Create a temporary WAV file to store the audio data # Create a temporary WAV file to store the audio data
temp_dir = tempfile.gettempdir() temp_dir = tempfile.gettempdir()
wav_path = os.path.join(temp_dir, f"audio_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav") wav_path = os.path.join(temp_dir, f"audio_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
wav_file = wave.open(wav_path, 'wb') wav_file = wave.open(wav_path, 'wb')
wav_file.setnchannels(CHANNELS) wav_file.setnchannels(self.CHANNELS)
wav_file.setsampwidth(p.get_sample_size(FORMAT)) wav_file.setsampwidth(p.get_sample_size(self.FORMAT))
wav_file.setframerate(RATE) wav_file.setframerate(self.RATE)
while RECORDING: while self.RECORDING:
data = stream.read(CHUNK, exception_on_overflow=False) data = stream.read(self.CHUNK, exception_on_overflow=False)
wav_file.writeframes(data) wav_file.writeframes(data)
wav_file.close() wav_file.close()
@ -173,7 +180,7 @@ class Device:
stream.close() stream.close()
print("Recording stopped.") print("Recording stopped.")
duration = wav_file.getnframes() / RATE duration = wav_file.getnframes() / self.RATE
if duration < 0.3: if duration < 0.3:
# Just pressed it. Send stop message # Just pressed it. Send stop message
if os.getenv('STT_RUNNER') == "client": if os.getenv('STT_RUNNER') == "client":
@ -198,10 +205,10 @@ class Device:
else: else:
# Stream audio # Stream audio
with open(wav_path, 'rb') as audio_file: with open(wav_path, 'rb') as audio_file:
byte_data = audio_file.read(CHUNK) byte_data = audio_file.read(self.CHUNK)
while byte_data: while byte_data:
send_queue.put(byte_data) send_queue.put(byte_data)
byte_data = audio_file.read(CHUNK) byte_data = audio_file.read(self.CHUNK)
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "end": True}) send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "end": True})
if os.path.exists(wav_path): if os.path.exists(wav_path):
@ -209,15 +216,14 @@ class Device:
def toggle_recording(self, state): def toggle_recording(self, state):
"""Toggle the recording state.""" """Toggle the recording state."""
global RECORDING, SPACEBAR_PRESSED if state and not self.SPACEBAR_PRESSED:
if state and not SPACEBAR_PRESSED: self.SPACEBAR_PRESSED = True
SPACEBAR_PRESSED = True if not self.RECORDING:
if not RECORDING: self.RECORDING = True
RECORDING = True
threading.Thread(target=self.record_audio).start() threading.Thread(target=self.record_audio).start()
elif not state and SPACEBAR_PRESSED: elif not state and self.SPACEBAR_PRESSED:
SPACEBAR_PRESSED = False self.SPACEBAR_PRESSED = False
RECORDING = False self.RECORDING = False
def on_press(self, key): def on_press(self, key):
"""Detect spacebar press and Ctrl+C combination.""" """Detect spacebar press and Ctrl+C combination."""
@ -254,58 +260,70 @@ class Device:
show_connection_log = True show_connection_log = True
while True: while True:
try: try:
async with websockets.connect(WS_URL) as websocket: print("Attempting to connect to the WS server...")
try:
ws = wsc.create_connection(WS_URL, timeout=5)
except wsc.WebSocketTimeoutException:
print("Timeout while trying to connect to the WebSocket server.")
continue
print("Connected to the WS server.")
if CAMERA_ENABLED: if CAMERA_ENABLED:
print("\nHold the spacebar to start recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.") print("\nHold the spacebar to start recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.")
else: else:
print("\nHold the spacebar to start recording. Press CTRL-C to exit.") print("\nHold the spacebar to start recording. Press CTRL-C to exit.")
asyncio.create_task(self.message_sender(websocket)) asyncio.create_task(self.message_sender(ws))
while True: while True:
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
chunk = await websocket.recv() chunk = await ws.recv()
logger.debug(f"Got this message from the server: {type(chunk)} {chunk}") logger.debug(f"Got this message from the server: {type(chunk)} {chunk}")
if type(chunk) == str: if isinstance(chunk, str):
chunk = json.loads(chunk) chunk = json.loads(chunk)
message = accumulator.accumulate(chunk) message = accumulator.accumulate(chunk)
if message == None: if message is None:
# Will be None until we have a full message ready # Will be None until we have a full message ready
continue continue
# At this point, we have our message # At this point, we have our message
if message["type"] == "audio" and message["format"].startswith("bytes"): if message["type"] == "audio" and message["format"].startswith("bytes"):
# Convert bytes to audio file # Convert bytes to audio file
audio_bytes = message["content"] audio_bytes = message["content"]
# Create an AudioSegment instance with the raw data # Create an AudioSegment instance with the raw data
audio = AudioSegment( audio = AudioSegment(
# raw audio data (bytes)
data=audio_bytes, data=audio_bytes,
# signed 16-bit little-endian format sample_width=AUDIO_SAMPLE_WIDTH,
sample_width=2, frame_rate=AUDIO_FRAME_RATE,
# 16,000 Hz frame rate channels=AUDIO_MONO_CHANNEL
frame_rate=16000,
# mono sound
channels=1
) )
self.audiosegments.append(audio) self.audiosegments.append(audio)
# Run the code if that's the client's job # Run the code if that's the client's job
if os.getenv('CODE_RUNNER') == "client": if os.getenv('CODE_RUNNER') == AUDIO_CODE_RUNNER_CLIENT:
if message["type"] == "code" and "end" in message: if message["type"] == "code" and "end" in message:
language = message["format"] language = message["format"]
code = message["content"] code = message["content"]
result = interpreter.computer.run(language, code) result = interpreter.computer.run(language, code)
send_queue.put(result) send_queue.put(result)
except:
except wsc.WebSocketConnectionClosedException:
print("WebSocket connection closed unexpectedly.")
if show_connection_log:
logger.info(f"Reconnecting to `{WS_URL}`...")
show_connection_log = False
await asyncio.sleep(2)
except wsc.WebSocketAddressException:
print(f"Invalid WebSocket URI: `{WS_URL}`. Please check the URI and try again.")
break # Exit the loop as the URI is invalid and retrying won't help
except Exception as e:
logger.debug(traceback.format_exc()) logger.debug(traceback.format_exc())
if show_connection_log: if show_connection_log:
logger.info(f"Connecting to `{WS_URL}`...") logger.info(f"Connecting to `{WS_URL}`...")
@ -320,13 +338,14 @@ class Device:
# Start watching the kernel if it's your job to do that # Start watching the kernel if it's your job to do that
if os.getenv('CODE_RUNNER') == "client": if os.getenv('CODE_RUNNER') == "client":
asyncio.create_task(put_kernel_messages_into_queue(send_queue)) kernel_checker = KernelChecker()
asyncio.create_task(put_kernel_messages_into_queue(kernel_checker, send_queue))
asyncio.create_task(self.play_audiosegments()) asyncio.create_task(self.play_audiosegments())
# If Raspberry Pi, add the button listener, otherwise use the spacebar # If Raspberry Pi, add the button listener, otherwise use the spacebar
if current_platform.startswith("raspberry-pi"): if current_platform.startswith("raspberry-pi"):
logger.info("Raspberry Pi detected, using button on GPIO pin 15") print("Raspberry Pi detected, using button on GPIO pin 15")
# Use GPIO pin 15 # Use GPIO pin 15
pindef = ["gpiochip4", "15"] # gpiofind PIN15 pindef = ["gpiochip4", "15"] # gpiofind PIN15
print("PINDEF", pindef) print("PINDEF", pindef)

0
software/source/clients/windows/__init__.py
Unescape View File

10
software/source/clients/windows/device.py
Unescape View File

@ -0,0 +1,10 @@
from ..base_device import Device
device = Device()
def main(server_url):
device.server_url = server_url
device.start()
if __name__ == "__main__":
main()

5
software/source/server/server.py
Unescape View File

@ -16,7 +16,7 @@ from starlette.websockets import WebSocket, WebSocketDisconnect
from pathlib import Path from pathlib import Path
import asyncio import asyncio
import urllib.parse import urllib.parse
from .utils.kernel import put_kernel_messages_into_queue from .utils.kernel import KernelChecker, put_kernel_messages_into_queue
from .i import configure_interpreter from .i import configure_interpreter
from interpreter import interpreter from interpreter import interpreter
from ..utils.accumulator import Accumulator from ..utils.accumulator import Accumulator
@ -420,7 +420,8 @@ async def main(server_host, server_port, llm_service, model, llm_supports_vision
# Start watching the kernel if it's your job to do that # Start watching the kernel if it's your job to do that
if True: # in the future, code can run on device. for now, just server. if True: # in the future, code can run on device. for now, just server.
asyncio.create_task(put_kernel_messages_into_queue(from_computer)) kernel_checker = KernelChecker()
asyncio.create_task(put_kernel_messages_into_queue(kernel_checker, from_computer))
config = Config(app, host=server_host, port=int(server_port), lifespan='on') config = Config(app, host=server_host, port=int(server_port), lifespan='on')
server = Server(config) server = Server(config)

73
software/source/server/utils/kernel.py
Unescape View File

@ -1,63 +1,74 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
import asyncio import asyncio
import subprocess import subprocess
import platform import platform
from dotenv import load_dotenv
from .logs import setup_logging, logger
from .logs import setup_logging load_dotenv() # take environment variables from .env.
from .logs import logger
setup_logging() setup_logging()
def get_kernel_messages(): class KernelChecker:
def __init__(self):
self._last_messages = ""
def get_kernel_messages(self):
""" """
Is this the way to do this? Fetch system logs or kernel message from the operating system.
- For MacOS, it uses syslog.
- For Linux, it uses dmesg.
- For Windows, it uses wevtutil with the 'qe' (query events) from the 'System' log
with the '/f:text' (format text) flag.
""" """
current_platform = platform.system() current_platform = platform.system().lower()
if current_platform == "Darwin": if current_platform == "darwin":
process = subprocess.Popen(['syslog'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) process = subprocess.Popen(['syslog'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate() output, _ = process.communicate()
return output.decode('utf-8') return output.decode('utf-8')
elif current_platform == "Linux": elif current_platform == "linux":
with open('/var/log/dmesg', 'r') as file: with open('/var/log/dmesg', 'r') as file:
return file.read() return file.read()
elif current_platform == "windows":
process = subprocess.Popen(['wevtutil', 'qe', 'System', '/f:text'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate()
try:
return output.decode('utf-8')
except UnicodeDecodeError:
try:
return output.decode('utf-16')
except UnicodeDecodeError:
return output.decode('cp1252')
else: else:
logger.info("Unsupported platform.") logger.info("Unsupported platform.")
return ""
def custom_filter(message): def custom_filter(self, message):
# Check for {TO_INTERPRETER{ message here }TO_INTERPRETER} pattern
if '{TO_INTERPRETER{' in message and '}TO_INTERPRETER}' in message: if '{TO_INTERPRETER{' in message and '}TO_INTERPRETER}' in message:
start = message.find('{TO_INTERPRETER{') + len('{TO_INTERPRETER{') start = message.find('{TO_INTERPRETER{') + len('{TO_INTERPRETER{')
end = message.find('}TO_INTERPRETER}', start) end = message.find('}TO_INTERPRETER}', start)
return message[start:end] return message[start:end]
# Check for USB mention
# elif 'USB' in message:
# return message
# # Check for network related keywords
# elif any(keyword in message for keyword in ['network', 'IP', 'internet', 'LAN', 'WAN', 'router', 'switch']) and "networkStatusForFlags" not in message:
# return message
else: else:
return None return None
last_messages = "" def check_filtered_kernel(self):
try:
def check_filtered_kernel(): messages = self.get_kernel_messages()
messages = get_kernel_messages() messages = messages.replace(self._last_messages, "")
messages.replace(last_messages, "")
messages = messages.split("\n") messages = messages.split("\n")
filtered_messages = [] filtered_messages = [message for message in messages if self.custom_filter(message)]
for message in messages:
if custom_filter(message): self._last_messages = "\n".join(filtered_messages)
filtered_messages.append(message) return self._last_messages
except Exception as e:
logger.error(f"Error while checking kernel messages: {e}")
return None
return "\n".join(filtered_messages)
async def put_kernel_messages_into_queue(queue): async def put_kernel_messages_into_queue(kernel_checker, queue):
while True: while True:
text = check_filtered_kernel() text = kernel_checker.check_filtered_kernel()
if text: if text:
if isinstance(queue, asyncio.Queue): if isinstance(queue, asyncio.Queue):
await queue.put({"role": "computer", "type": "console", "start": True}) await queue.put({"role": "computer", "type": "console", "start": True})

10
software/start.py
Unescape View File

@ -15,7 +15,7 @@ app = typer.Typer()
@app.command() @app.command()
def run( def run(
server: bool = typer.Option(False, "--server", help="Run server"), server: bool = typer.Option(False, "--server", help="Run server"),
server_host: str = typer.Option("0.0.0.0", "--server-host", help="Specify the server host where the server will deploy"), server_host: str = typer.Option("127.0.0.1", "--server-host", help="Specify the server host where the server will deploy"),
server_port: int = typer.Option(10001, "--server-port", help="Specify the server port where the server will deploy"), server_port: int = typer.Option(10001, "--server-port", help="Specify the server port where the server will deploy"),
tunnel_service: str = typer.Option("ngrok", "--tunnel-service", help="Specify the tunnel service"), tunnel_service: str = typer.Option("ngrok", "--tunnel-service", help="Specify the tunnel service"),
@ -120,10 +120,12 @@ def _run(
if client: if client:
if client_type == "auto": if client_type == "auto":
system_type = platform.system() system_type = platform.system().lower()
if system_type == "Darwin": # Mac OS if system_type == "darwin": # Mac OS
client_type = "mac" client_type = "mac"
elif system_type == "Linux": # Linux System elif system_type == "windows":
client_type = "windows"
elif system_type == "linux": # Linux System
try: try:
with open('/proc/device-tree/model', 'r') as m: with open('/proc/device-tree/model', 'r') as m:
if 'raspberry pi' in m.read().lower(): if 'raspberry pi' in m.read().lower():

Write Preview
Loading…
Cancel
Save