Fixed structure

pull/72/head
Tahir Siddiuqe 1 year ago
parent 2e5c3f59f1
commit f67d64c091

@ -27,7 +27,7 @@ jobs:
run: sudo apt-get install portaudio19-dev python-all-dev run: sudo apt-get install portaudio19-dev python-all-dev
- name: Install Poetry Package - name: Install Poetry Package
working-directory: 01OS # Set the working directory to "01OS" working-directory: _01OS # Set the working directory to "_01OS"
run: | run: |
pip install --upgrade pip pip install --upgrade pip
pip install poetry==1.3.2 pip install poetry==1.3.2

@ -21,7 +21,7 @@ jobs:
defaults: defaults:
run: run:
working-directory: 01OS working-directory: _01OS
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}

8
.gitignore vendored

@ -4,9 +4,9 @@ __pycache__/
*.py[cod] *.py[cod]
*$py.class *$py.class
01OS/01OS/server/conversations/user.json _01OS/_01OS/server/conversations/user.json
01OS/01OS/server/tts/local_service/* _01OS/_01OS/server/tts/local_service/*
01OS/01OS/server/stt/local_service/* _01OS/_01OS/server/stt/local_service/*
# C extensions # C extensions
*.so *.so
@ -167,5 +167,5 @@ cython_debug/
# ignore the aifs index files # ignore the aifs index files
_.aifs _.aifs
01OS/output_audio.wav _01OS/output_audio.wav
.DS_Store .DS_Store

@ -19,7 +19,7 @@ repos:
hooks: hooks:
- id: pytest - id: pytest
name: pytest name: pytest
entry: pytest 01OS/tests entry: pytest _01OS/tests
language: system language: system
types: [python] types: [python]
pass_filenames: false pass_filenames: false

@ -0,0 +1,7 @@
{
"python.testing.pytestArgs": [
"_01OS"
],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true
}

@ -1,4 +0,0 @@
_archive
__pycache__
.idea

@ -1,9 +0,0 @@
The open-source language model computer.
```bash
pip install 01OS
```
```bash
01 # Runs the 01 server and client
```

@ -1,351 +0,0 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
import os
import asyncio
import threading
import os
import pyaudio
from starlette.websockets import WebSocket
from queue import Queue
from pynput import keyboard
import json
import traceback
import websockets
import queue
import pydub
import ast
from pydub import AudioSegment
from pydub.playback import play
import io
import time
import wave
import tempfile
from datetime import datetime
import cv2
import base64
from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run?
# In the future, I guess kernel watching code should be elsewhere? Somewhere server / client agnostic?
from ..server.utils.kernel import put_kernel_messages_into_queue
from ..server.utils.get_system_info import get_system_info
from ..server.utils.process_utils import kill_process_tree
from ..server.utils.logs import setup_logging
from ..server.utils.logs import logger
setup_logging()
os.environ["STT_RUNNER"] = "server"
os.environ["TTS_RUNNER"] = "server"
from ..utils.accumulator import Accumulator
accumulator = Accumulator()
# Configuration for Audio Recording
CHUNK = 1024 # Record in chunks of 1024 samples
FORMAT = pyaudio.paInt16 # 16 bits per sample
CHANNELS = 1 # Mono
RATE = 44100 # Sample rate
RECORDING = False # Flag to control recording state
SPACEBAR_PRESSED = False # Flag to track spacebar press state
# Camera configuration
CAMERA_ENABLED = os.getenv('CAMERA_ENABLED', False)
if type(CAMERA_ENABLED) == str:
CAMERA_ENABLED = (CAMERA_ENABLED.lower() == "true")
CAMERA_DEVICE_INDEX = int(os.getenv('CAMERA_DEVICE_INDEX', 0))
CAMERA_WARMUP_SECONDS = float(os.getenv('CAMERA_WARMUP_SECONDS', 0))
# Specify OS
current_platform = get_system_info()
# Initialize PyAudio
p = pyaudio.PyAudio()
send_queue = queue.Queue()
class Device:
def __init__(self):
self.pressed_keys = set()
self.captured_images = []
self.audiosegments = []
self.server_url = ""
def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX):
"""Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list."""
image_path = None
cap = cv2.VideoCapture(camera_index)
ret, frame = cap.read() # Capture a single frame to initialize the camera
if CAMERA_WARMUP_SECONDS > 0:
# Allow camera to warm up, then snap a picture again
# This is a workaround for some cameras that don't return a properly exposed
# picture immediately when they are first turned on
time.sleep(CAMERA_WARMUP_SECONDS)
ret, frame = cap.read()
if ret:
temp_dir = tempfile.gettempdir()
image_path = os.path.join(temp_dir, f"01_photo_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.png")
self.captured_images.append(image_path)
cv2.imwrite(image_path, frame)
logger.info(f"Camera image captured to {image_path}")
logger.info(f"You now have {len(self.captured_images)} images which will be sent along with your next audio message.")
else:
logger.error(f"Error: Couldn't capture an image from camera ({camera_index})")
cap.release()
return image_path
def encode_image_to_base64(self, image_path):
"""Encodes an image file to a base64 string."""
with open(image_path, "rb") as image_file:
return base64.b64encode(image_file.read()).decode('utf-8')
def add_image_to_send_queue(self, image_path):
"""Encodes an image and adds an LMC message to the send queue with the image data."""
base64_image = self.encode_image_to_base64(image_path)
image_message = {
"role": "user",
"type": "image",
"format": "base64.png",
"content": base64_image
}
send_queue.put(image_message)
# Delete the image file from the file system after sending it
os.remove(image_path)
def queue_all_captured_images(self):
"""Queues all captured images to be sent."""
for image_path in self.captured_images:
self.add_image_to_send_queue(image_path)
self.captured_images.clear() # Clear the list after sending
async def play_audiosegments(self):
"""Plays them sequentially."""
while True:
try:
for audio in self.audiosegments:
play(audio)
self.audiosegments.remove(audio)
await asyncio.sleep(0.1)
except asyncio.exceptions.CancelledError:
# This happens once at the start?
pass
except:
logger.info(traceback.format_exc())
def record_audio(self):
if os.getenv('STT_RUNNER') == "server":
# STT will happen on the server. we're sending audio.
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "start": True})
elif os.getenv('STT_RUNNER') == "client":
# STT will happen here, on the client. we're sending text.
send_queue.put({"role": "user", "type": "message", "start": True})
else:
raise Exception("STT_RUNNER must be set to either 'client' or 'server'.")
"""Record audio from the microphone and add it to the queue."""
stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=CHUNK)
logger.info("Recording started...")
global RECORDING
# Create a temporary WAV file to store the audio data
temp_dir = tempfile.gettempdir()
wav_path = os.path.join(temp_dir, f"audio_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
wav_file = wave.open(wav_path, 'wb')
wav_file.setnchannels(CHANNELS)
wav_file.setsampwidth(p.get_sample_size(FORMAT))
wav_file.setframerate(RATE)
while RECORDING:
data = stream.read(CHUNK, exception_on_overflow=False)
wav_file.writeframes(data)
wav_file.close()
stream.stop_stream()
stream.close()
logger.info("Recording stopped.")
duration = wav_file.getnframes() / RATE
if duration < 0.3:
# Just pressed it. Send stop message
if os.getenv('STT_RUNNER') == "client":
send_queue.put({"role": "user", "type": "message", "content": "stop"})
send_queue.put({"role": "user", "type": "message", "end": True})
else:
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "content": ""})
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "end": True})
else:
self.queue_all_captured_images()
if os.getenv('STT_RUNNER') == "client":
# THIS DOES NOT WORK. We moved to this very cool stt_service, llm_service
# way of doing things. stt_wav is not a thing anymore. Needs work to work
# Run stt then send text
text = stt_wav(wav_path)
logger.debug(f"STT result: {text}")
send_queue.put({"role": "user", "type": "message", "content": text})
send_queue.put({"role": "user", "type": "message", "end": True})
else:
# Stream audio
with open(wav_path, 'rb') as audio_file:
byte_data = audio_file.read(CHUNK)
while byte_data:
send_queue.put(byte_data)
byte_data = audio_file.read(CHUNK)
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "end": True})
if os.path.exists(wav_path):
os.remove(wav_path)
def toggle_recording(self, state):
"""Toggle the recording state."""
global RECORDING, SPACEBAR_PRESSED
if state and not SPACEBAR_PRESSED:
SPACEBAR_PRESSED = True
if not RECORDING:
RECORDING = True
threading.Thread(target=self.record_audio).start()
elif not state and SPACEBAR_PRESSED:
SPACEBAR_PRESSED = False
RECORDING = False
def on_press(self, key):
"""Detect spacebar press and Ctrl+C combination."""
self.pressed_keys.add(key) # Add the pressed key to the set
if keyboard.Key.space in self.pressed_keys:
self.toggle_recording(True)
elif {keyboard.Key.ctrl, keyboard.KeyCode.from_char('c')} <= self.pressed_keys:
logger.info("Ctrl+C pressed. Exiting...")
kill_process_tree()
os._exit(0)
def on_release(self, key):
"""Detect spacebar release and 'c' key press for camera, and handle key release."""
self.pressed_keys.discard(key) # Remove the released key from the key press tracking set
if key == keyboard.Key.space:
self.toggle_recording(False)
elif CAMERA_ENABLED and key == keyboard.KeyCode.from_char('c'):
self.fetch_image_from_camera()
async def message_sender(self, websocket):
while True:
message = await asyncio.get_event_loop().run_in_executor(None, send_queue.get)
if isinstance(message, bytes):
await websocket.send(message)
else:
await websocket.send(json.dumps(message))
send_queue.task_done()
await asyncio.sleep(0.01)
async def websocket_communication(self, WS_URL):
while True:
try:
async with websockets.connect(WS_URL) as websocket:
if CAMERA_ENABLED:
logger.info("Press the spacebar to start/stop recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.")
else:
logger.info("Press the spacebar to start/stop recording. Press CTRL-C to exit.")
asyncio.create_task(self.message_sender(websocket))
while True:
await asyncio.sleep(0.01)
chunk = await websocket.recv()
logger.debug(f"Got this message from the server: {type(chunk)} {chunk}")
if type(chunk) == str:
chunk = json.loads(chunk)
message = accumulator.accumulate(chunk)
if message == None:
# Will be None until we have a full message ready
continue
# At this point, we have our message
if message["type"] == "audio" and message["format"].startswith("bytes"):
# Convert bytes to audio file
audio_bytes = message["content"]
# Create an AudioSegment instance with the raw data
audio = AudioSegment(
# raw audio data (bytes)
data=audio_bytes,
# signed 16-bit little-endian format
sample_width=2,
# 16,000 Hz frame rate
frame_rate=16000,
# mono sound
channels=1
)
self.audiosegments.append(audio)
# Run the code if that's the client's job
if os.getenv('CODE_RUNNER') == "client":
if message["type"] == "code" and "end" in message:
language = message["format"]
code = message["content"]
result = interpreter.computer.run(language, code)
send_queue.put(result)
except:
logger.debug(traceback.format_exc())
logger.info(f"Connecting to `{WS_URL}`...")
await asyncio.sleep(2)
async def start_async(self):
# Configuration for WebSocket
WS_URL = f"ws://{self.server_url}"
# Start the WebSocket communication
asyncio.create_task(self.websocket_communication(WS_URL))
# Start watching the kernel if it's your job to do that
if os.getenv('CODE_RUNNER') == "client":
asyncio.create_task(put_kernel_messages_into_queue(send_queue))
asyncio.create_task(self.play_audiosegments())
# If Raspberry Pi, add the button listener, otherwise use the spacebar
if current_platform.startswith("raspberry-pi"):
logger.info("Raspberry Pi detected, using button on GPIO pin 15")
# Use GPIO pin 15
pindef = ["gpiochip4", "15"] # gpiofind PIN15
print("PINDEF", pindef)
# HACK: needs passwordless sudo
process = await asyncio.create_subprocess_exec("sudo", "gpiomon", "-brf", *pindef, stdout=asyncio.subprocess.PIPE)
while True:
line = await process.stdout.readline()
if line:
line = line.decode().strip()
if "FALLING" in line:
self.toggle_recording(False)
elif "RISING" in line:
self.toggle_recording(True)
else:
break
else:
# Keyboard listener for spacebar press/release
listener = keyboard.Listener(on_press=self.on_press, on_release=self.on_release)
listener.start()
def start(self):
if os.getenv('TEACH_MODE') != "True":
asyncio.run(self.start_async())
p.terminate()

@ -1,11 +0,0 @@
# ESP32 Playback
To set up audio recording + playback on the ESP32 (M5 Atom), do the following:
1. Open Arduino IDE, and open the `client/client.ino` file
2. Go to Tools -> Board -> Boards Manager, search "esp32", then install the boards by Arduino and Espressif
3. Go to Tools -> Manage Libraries, then install the following:
- M5Atom by M5Stack
- WebSockets by Markus Sattler
4. The board needs to connect to WiFi. Once you flash, connect to ESP32 wifi "captive" which will get wifi details. Once it connects, it will ask you to enter 01OS server address in the format "domain.com:port" or "ip:port". Once its able to connect you can use the device.
5. To flash the .ino to the board, connect the board to the USB port, select the port from the dropdown on the IDE, then select the M5Atom board (or M5Stack-ATOM if you have that). Click on upload to flash the board.

@ -1,675 +0,0 @@
#include <driver/i2s.h>
#include <M5Atom.h>
#include <Arduino.h> //not needed in the arduino ide
#include <AsyncTCP.h> //https://github.com/me-no-dev/AsyncTCP using the latest dev version from @me-no-dev
#include <DNSServer.h>
#include <ESPAsyncWebServer.h> //https://github.com/me-no-dev/ESPAsyncWebServer using the latest dev version from @me-no-dev
#include <esp_wifi.h> //Used for mpdu_rx_disable android workaround
#include <ArduinoHttpClient.h>
#include <WiFi.h>
#include <WiFiClient.h>
#include <WiFiMulti.h>
#include <WiFiClientSecure.h>
#include <WebSocketsClient.h>
String server_domain = "";
int server_port = 8000;
// ----------------------- START OF WIFI CAPTIVE PORTAL -------------------
// Pre reading on the fundamentals of captive portals https://textslashplain.com/2022/06/24/captive-portals/
const char *ssid = "captive"; // FYI The SSID can't have a space in it.
// const char * password = "12345678"; //Atleast 8 chars
const char *password = NULL; // no password
#define MAX_CLIENTS 4 // ESP32 supports up to 10 but I have not tested it yet
#define WIFI_CHANNEL 6 // 2.4ghz channel 6 https://en.wikipedia.org/wiki/List_of_WLAN_channels#2.4_GHz_(802.11b/g/n/ax)
const IPAddress localIP(4, 3, 2, 1); // the IP address the web server, Samsung requires the IP to be in public space
const IPAddress gatewayIP(4, 3, 2, 1); // IP address of the network should be the same as the local IP for captive portals
const IPAddress subnetMask(255, 255, 255, 0); // no need to change: https://avinetworks.com/glossary/subnet-mask/
const String localIPURL = "http://4.3.2.1"; // a string version of the local IP with http, used for redirecting clients to your webpage
// Number of milliseconds to wait without receiving any data before we give up
const int kNetworkTimeout = 30 * 1000;
// Number of milliseconds to wait if no data is available before trying again
const int kNetworkDelay = 1000;
String generateHTMLWithSSIDs()
{
String html = "<!DOCTYPE html><html><body><h2>Select Wi-Fi Network</h2><form action='/submit' method='POST'><label for='ssid'>SSID:</label><select id='ssid' name='ssid'>";
int n = WiFi.scanComplete();
for (int i = 0; i < n; ++i)
{
html += "<option value='" + WiFi.SSID(i) + "'>" + WiFi.SSID(i) + "</option>";
}
html += "</select><br><label for='password'>Password:</label><input type='password' id='password' name='password'><br><input type='submit' value='Connect'></form></body></html>";
return html;
}
const char index_html[] PROGMEM = R"=====(
<!DOCTYPE html>
<html>
<head>
<title>WiFi Setup</title>
<style>
body {background-color:#06cc13;}
h1 {color: white;}
h2 {color: white;}
</style>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<body>
<h1>WiFi Setup</h1>
<form action="/submit" method="post">
<label for="ssid">SSID:</label><br>
<input type="text" id="ssid" name="ssid"><br>
<label for="password">Password:</label><br>
<input type="password" id="password" name="password"><br><br>
<input type="submit" value="Connect">
</form>
</body>
</html>
)=====";
const char post_connected_html[] PROGMEM = R"=====(
<!DOCTYPE html>
<html>
<head>
<title>01OS Setup</title>
<style>
body {background-color:white;}
h1 {color: black;}
h2 {color: black;}
</style>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<body>
<h1>01OS Setup</h1>
<form action="/submit_01os" method="post">
<label for="server_address">01OS Server Address:</label><br>
<input type="text" id="server_address" name="server_address"><br>
<input type="submit" value="Connect">
</form>
</body>
</html>
)=====";
DNSServer dnsServer;
AsyncWebServer server(80);
void setUpDNSServer(DNSServer &dnsServer, const IPAddress &localIP)
{
// Define the DNS interval in milliseconds between processing DNS requests
#define DNS_INTERVAL 30
// Set the TTL for DNS response and start the DNS server
dnsServer.setTTL(3600);
dnsServer.start(53, "*", localIP);
}
void startSoftAccessPoint(const char *ssid, const char *password, const IPAddress &localIP, const IPAddress &gatewayIP)
{
// Define the maximum number of clients that can connect to the server
#define MAX_CLIENTS 4
// Define the WiFi channel to be used (channel 6 in this case)
#define WIFI_CHANNEL 6
// Set the WiFi mode to access point and station
// WiFi.mode(WIFI_MODE_AP);
// Define the subnet mask for the WiFi network
const IPAddress subnetMask(255, 255, 255, 0);
// Configure the soft access point with a specific IP and subnet mask
WiFi.softAPConfig(localIP, gatewayIP, subnetMask);
// Start the soft access point with the given ssid, password, channel, max number of clients
WiFi.softAP(ssid, password, WIFI_CHANNEL, 0, MAX_CLIENTS);
// Disable AMPDU RX on the ESP32 WiFi to fix a bug on Android
esp_wifi_stop();
esp_wifi_deinit();
wifi_init_config_t my_config = WIFI_INIT_CONFIG_DEFAULT();
my_config.ampdu_rx_enable = false;
esp_wifi_init(&my_config);
esp_wifi_start();
vTaskDelay(100 / portTICK_PERIOD_MS); // Add a small delay
}
void connectToWifi(String ssid, String password)
{
WiFi.begin(ssid.c_str(), password.c_str());
// Wait for connection to establish
int attempts = 0;
while (WiFi.status() != WL_CONNECTED && attempts < 20)
{
delay(1000);
Serial.print(".");
attempts++;
}
if (WiFi.status() == WL_CONNECTED)
{
Serial.println("Connected to Wi-Fi");
}
else
{
Serial.println("Failed to connect to Wi-Fi. Check credentials.");
}
}
bool connectTo01OS(String server_address)
{
int err = 0;
String domain;
String portStr;
if (server_address.indexOf(":") != -1) {
domain = server_address.substring(0, server_address.indexOf(':'));
portStr = server_address.substring(server_address.indexOf(':') + 1);
} else {
domain = server_address;
portStr = ""; // or any default value you want to assign
}
int port = 0; // Default port value
if (portStr.length() > 0) {
port = portStr.toInt();
}
WiFiClient c;
HttpClient http(c, domain.c_str(), port);
Serial.println("Connecting to 01OS at " + domain + ":" + port + "/ping");
err = http.get("/ping");
// err = http.get("arduino.cc", "/");
bool connectionSuccess = false;
if (err == 0)
{
Serial.println("Started the ping request");
err = http.responseStatusCode();
if (err >= 0)
{
Serial.print("Got status code: ");
Serial.println(err);
if (err == 200)
{
server_domain = domain;
server_port = port;
connectionSuccess = true;
}
err = http.skipResponseHeaders();
if (err >= 0)
{
int bodyLen = http.contentLength();
Serial.print("Content length is: ");
Serial.println(bodyLen);
Serial.println();
Serial.println("Body returned follows:");
// Now we've got to the body, so we can print it out
unsigned long timeoutStart = millis();
char c;
// Whilst we haven't timed out & haven't reached the end of the body
while ((http.connected() || http.available()) &&
((millis() - timeoutStart) < kNetworkTimeout))
{
if (http.available())
{
c = http.read();
// Print out this character
Serial.print(c);
bodyLen--;
// We read something, reset the timeout counter
timeoutStart = millis();
}
else
{
// We haven't got any data, so let's pause to allow some to
// arrive
delay(kNetworkDelay);
}
}
}
else
{
Serial.print("Failed to skip response headers: ");
Serial.println(err);
}
}
else
{
Serial.print("Getting response failed: ");
Serial.println(err);
}
}
else
{
Serial.print("Connect failed: ");
Serial.println(err);
}
return connectionSuccess;
}
void setUpWebserver(AsyncWebServer &server, const IPAddress &localIP)
{
//======================== Webserver ========================
// WARNING IOS (and maybe macos) WILL NOT POP UP IF IT CONTAINS THE WORD "Success" https://www.esp8266.com/viewtopic.php?f=34&t=4398
// SAFARI (IOS) IS STUPID, G-ZIPPED FILES CAN'T END IN .GZ https://github.com/homieiot/homie-esp8266/issues/476 this is fixed by the webserver serve static function.
// SAFARI (IOS) there is a 128KB limit to the size of the HTML. The HTML can reference external resources/images that bring the total over 128KB
// SAFARI (IOS) popup browser has some severe limitations (javascript disabled, cookies disabled)
// Required
server.on("/connecttest.txt", [](AsyncWebServerRequest *request)
{ request->redirect("http://logout.net"); }); // windows 11 captive portal workaround
server.on("/wpad.dat", [](AsyncWebServerRequest *request)
{ request->send(404); }); // Honestly don't understand what this is but a 404 stops win 10 keep calling this repeatedly and panicking the esp32 :)
// Background responses: Probably not all are Required, but some are. Others might speed things up?
// A Tier (commonly used by modern systems)
server.on("/generate_204", [](AsyncWebServerRequest *request)
{ request->redirect(localIPURL); }); // android captive portal redirect
server.on("/redirect", [](AsyncWebServerRequest *request)
{ request->redirect(localIPURL); }); // microsoft redirect
server.on("/hotspot-detect.html", [](AsyncWebServerRequest *request)
{ request->redirect(localIPURL); }); // apple call home
server.on("/canonical.html", [](AsyncWebServerRequest *request)
{ request->redirect(localIPURL); }); // firefox captive portal call home
server.on("/success.txt", [](AsyncWebServerRequest *request)
{ request->send(200); }); // firefox captive portal call home
server.on("/ncsi.txt", [](AsyncWebServerRequest *request)
{ request->redirect(localIPURL); }); // windows call home
// B Tier (uncommon)
// server.on("/chrome-variations/seed",[](AsyncWebServerRequest *request){request->send(200);}); //chrome captive portal call home
// server.on("/service/update2/json",[](AsyncWebServerRequest *request){request->send(200);}); //firefox?
// server.on("/chat",[](AsyncWebServerRequest *request){request->send(404);}); //No stop asking Whatsapp, there is no internet connection
// server.on("/startpage",[](AsyncWebServerRequest *request){request->redirect(localIPURL);});
// return 404 to webpage icon
server.on("/favicon.ico", [](AsyncWebServerRequest *request)
{ request->send(404); }); // webpage icon
// Serve Basic HTML Page
server.on("/", HTTP_ANY, [](AsyncWebServerRequest *request)
{
String htmlContent = index_html;
Serial.printf("wifi scan complete: %d . WIFI_SCAN_RUNNING: %d", WiFi.scanComplete(), WIFI_SCAN_RUNNING);
if(WiFi.scanComplete() > 0) {
// Scan complete, process results
Serial.println("done scanning wifi");
htmlContent = generateHTMLWithSSIDs();
// WiFi.scanNetworks(true); // Start a new scan in async mode
}
AsyncWebServerResponse *response = request->beginResponse(200, "text/html", htmlContent);
response->addHeader("Cache-Control", "public,max-age=31536000"); // save this file to cache for 1 year (unless you refresh)
request->send(response);
Serial.println("Served Basic HTML Page"); });
// the catch all
server.onNotFound([](AsyncWebServerRequest *request)
{
request->redirect(localIPURL);
Serial.print("onnotfound ");
Serial.print(request->host()); // This gives some insight into whatever was being requested on the serial monitor
Serial.print(" ");
Serial.print(request->url());
Serial.print(" sent redirect to " + localIPURL + "\n"); });
server.on("/submit", HTTP_POST, [](AsyncWebServerRequest *request)
{
String ssid;
String password;
// Check if SSID parameter exists and assign it
if(request->hasParam("ssid", true)) {
ssid = request->getParam("ssid", true)->value();
}
// Check if Password parameter exists and assign it
if(request->hasParam("password", true)) {
password = request->getParam("password", true)->value();
}
// Attempt to connect to the Wi-Fi network with these credentials
connectToWifi(ssid, password);
// Redirect user or send a response back
if (WiFi.status() == WL_CONNECTED) {
String htmlContent = post_connected_html;
AsyncWebServerResponse *response = request->beginResponse(200, "text/html", htmlContent);
response->addHeader("Cache-Control", "public,max-age=31536000"); // save this file to cache for 1 year (unless you refresh)
request->send(response);
Serial.println("Served Post connection HTML Page");
} else {
request->send(200, "text/plain", "Failed to connect to " + ssid);
} });
server.on("/submit_01os", HTTP_POST, [](AsyncWebServerRequest *request)
{
String server_address;
// Check if SSID parameter exists and assign it
if(request->hasParam("server_address", true)) {
server_address = request->getParam("server_address", true)->value();
}
// Attempt to connect to the Wi-Fi network with these credentials
bool connectedToServer = connectTo01OS(server_address);
// Redirect user or send a response back
String connectionMessage;
if (connectedToServer)
{
connectionMessage = "Connected to 01OS " + server_address;
}
else
{
connectionMessage = "Couldn't connect to 01OS " + server_address;
}
request->send(200, "text/plain", connectionMessage); });
}
// ----------------------- END OF WIFI CAPTIVE PORTAL -------------------
// ----------------------- START OF PLAYBACK -------------------
#define CONFIG_I2S_BCK_PIN 19
#define CONFIG_I2S_LRCK_PIN 33
#define CONFIG_I2S_DATA_PIN 22
#define CONFIG_I2S_DATA_IN_PIN 23
#define SPEAKER_I2S_NUMBER I2S_NUM_0
#define MODE_MIC 0
#define MODE_SPK 1
#define DATA_SIZE 1024
uint8_t microphonedata0[1024 * 10];
uint8_t speakerdata0[1024 * 1];
int speaker_offset;
int data_offset;
bool recording = false;
WebSocketsClient webSocket;
class ButtonChecker
{
public:
void loop()
{
lastTickState = thisTickState;
thisTickState = M5.Btn.isPressed() != 0;
}
bool justPressed()
{
return thisTickState && !lastTickState;
}
bool justReleased()
{
return !thisTickState && lastTickState;
}
private:
bool lastTickState = false;
bool thisTickState = false;
};
ButtonChecker button = ButtonChecker();
void InitI2SSpeakerOrMic(int mode)
{
Serial.printf("InitI2sSpeakerOrMic %d\n", mode);
esp_err_t err = ESP_OK;
i2s_driver_uninstall(SPEAKER_I2S_NUMBER);
i2s_config_t i2s_config = {
.mode = (i2s_mode_t)(I2S_MODE_MASTER),
.sample_rate = 16000,
.bits_per_sample =
I2S_BITS_PER_SAMPLE_16BIT, // is fixed at 12bit, stereo, MSB
.channel_format = I2S_CHANNEL_FMT_ALL_RIGHT,
#if ESP_IDF_VERSION > ESP_IDF_VERSION_VAL(4, 1, 0)
.communication_format =
I2S_COMM_FORMAT_STAND_I2S, // Set the format of the communication.
#else // 设置通讯格式
.communication_format = I2S_COMM_FORMAT_I2S,
#endif
.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1,
.dma_buf_count = 6,
.dma_buf_len = 60,
};
if (mode == MODE_MIC)
{
i2s_config.mode =
(i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX | I2S_MODE_PDM);
}
else
{
i2s_config.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_TX);
i2s_config.use_apll = false;
i2s_config.tx_desc_auto_clear = true;
}
err += i2s_driver_install(SPEAKER_I2S_NUMBER, &i2s_config, 0, NULL);
i2s_pin_config_t tx_pin_config;
#if (ESP_IDF_VERSION > ESP_IDF_VERSION_VAL(4, 3, 0))
tx_pin_config.mck_io_num = I2S_PIN_NO_CHANGE;
#endif
tx_pin_config.bck_io_num = CONFIG_I2S_BCK_PIN;
tx_pin_config.ws_io_num = CONFIG_I2S_LRCK_PIN;
tx_pin_config.data_out_num = CONFIG_I2S_DATA_PIN;
tx_pin_config.data_in_num = CONFIG_I2S_DATA_IN_PIN;
// Serial.println("Init i2s_set_pin");
err += i2s_set_pin(SPEAKER_I2S_NUMBER, &tx_pin_config);
// Serial.println("Init i2s_set_clk");
err += i2s_set_clk(SPEAKER_I2S_NUMBER, 16000, I2S_BITS_PER_SAMPLE_16BIT,
I2S_CHANNEL_MONO);
}
void speaker_play(uint8_t *payload, uint32_t len)
{
Serial.printf("received %lu bytes", len);
size_t bytes_written;
InitI2SSpeakerOrMic(MODE_SPK);
i2s_write(SPEAKER_I2S_NUMBER, payload, len,
&bytes_written, portMAX_DELAY);
}
void webSocketEvent(WStype_t type, uint8_t *payload, size_t length)
{
switch (type)
{
case WStype_DISCONNECTED:
Serial.printf("[WSc] Disconnected!\n");
break;
case WStype_CONNECTED:
Serial.printf("[WSc] Connected to url: %s\n", payload);
// send message to server when Connected
break;
case WStype_TEXT:
Serial.printf("[WSc] get text: %s\n", payload);
{
std::string str(payload, payload + length);
bool isAudio = str.find("\"audio\"") != std::string::npos;
if (isAudio && str.find("\"start\"") != std::string::npos)
{
Serial.println("start playback");
speaker_offset = 0;
InitI2SSpeakerOrMic(MODE_SPK);
}
else if (isAudio && str.find("\"end\"") != std::string::npos)
{
Serial.println("end playback");
// speaker_play(speakerdata0, speaker_offset);
// speaker_offset = 0;
}
}
// send message to server
// webSocket.sendTXT("message here");
break;
case WStype_BIN:
Serial.printf("[WSc] get binary length: %u\n", length);
memcpy(speakerdata0 + speaker_offset, payload, length);
speaker_offset += length;
size_t bytes_written;
i2s_write(SPEAKER_I2S_NUMBER, speakerdata0, speaker_offset, &bytes_written, portMAX_DELAY);
speaker_offset = 0;
// send data to server
// webSocket.sendBIN(payload, length);
break;
case WStype_ERROR:
case WStype_FRAGMENT_TEXT_START:
case WStype_FRAGMENT_BIN_START:
case WStype_FRAGMENT:
case WStype_FRAGMENT_FIN:
break;
}
}
void websocket_setup(String server_domain, int port)
{
if (WiFi.status() != WL_CONNECTED)
{
Serial.println("Not connected to WiFi. Abandoning setup websocket");
return;
}
Serial.println("connected to WiFi");
webSocket.begin(server_domain, port, "/");
webSocket.onEvent(webSocketEvent);
// webSocket.setAuthorization("user", "Password");
webSocket.setReconnectInterval(5000);
}
void flush_microphone()
{
Serial.printf("[microphone] flushing %d bytes of data\n", data_offset);
if (data_offset == 0)
return;
webSocket.sendBIN(microphonedata0, data_offset);
data_offset = 0;
}
// ----------------------- END OF PLAYBACK -------------------
bool hasSetupWebsocket = false;
void setup()
{
// Set the transmit buffer size for the Serial object and start it with a baud rate of 115200.
Serial.setTxBufferSize(1024);
Serial.begin(115200);
// Wait for the Serial object to become available.
while (!Serial)
;
WiFi.mode(WIFI_AP_STA);
// Print a welcome message to the Serial port.
Serial.println("\n\nCaptive Test, V0.5.0 compiled " __DATE__ " " __TIME__ " by CD_FER"); //__DATE__ is provided by the platformio ide
Serial.printf("%s-%d\n\r", ESP.getChipModel(), ESP.getChipRevision());
startSoftAccessPoint(ssid, password, localIP, gatewayIP);
setUpDNSServer(dnsServer, localIP);
WiFi.scanNetworks(true);
setUpWebserver(server, localIP);
server.begin();
Serial.print("\n");
Serial.print("Startup Time:"); // should be somewhere between 270-350 for Generic ESP32 (D0WDQ6 chip, can have a higher startup time on first boot)
Serial.println(millis());
Serial.print("\n");
M5.begin(true, false, true);
M5.dis.drawpix(0, CRGB(128, 128, 0));
}
void loop()
{
dnsServer.processNextRequest(); // I call this atleast every 10ms in my other projects (can be higher but I haven't tested it for stability)
delay(DNS_INTERVAL); // seems to help with stability, if you are doing other things in the loop this may not be needed
// Check WiFi connection status
if (WiFi.status() == WL_CONNECTED && !hasSetupWebsocket)
{
if (server_domain != "")
{
Serial.println("Setting up websocket to 01OS " + server_domain + ":" + server_port);
websocket_setup(server_domain, server_port);
InitI2SSpeakerOrMic(MODE_SPK);
hasSetupWebsocket = true;
Serial.println("Websocket connection flow completed");
}
else
{
Serial.println("No valid 01OS server address yet...");
}
// If connected, you might want to do something, like printing the IP address
// Serial.println("Connected to WiFi!");
// Serial.println("IP Address: " + WiFi.localIP().toString());
// Serial.println("SSID " + WiFi.SSID());
}
if (WiFi.status() == WL_CONNECTED && hasSetupWebsocket)
{
button.loop();
if (button.justPressed())
{
Serial.println("Recording...");
webSocket.sendTXT("{\"role\": \"user\", \"type\": \"audio\", \"format\": \"bytes.raw\", \"start\": true}");
InitI2SSpeakerOrMic(MODE_MIC);
recording = true;
data_offset = 0;
Serial.println("Recording ready.");
}
else if (button.justReleased())
{
Serial.println("Stopped recording.");
webSocket.sendTXT("{\"role\": \"user\", \"type\": \"audio\", \"format\": \"bytes.raw\", \"end\": true}");
flush_microphone();
recording = false;
data_offset = 0;
}
else if (recording)
{
Serial.printf("Reading chunk at %d...\n", data_offset);
size_t bytes_read;
i2s_read(
SPEAKER_I2S_NUMBER,
(char *)(microphonedata0 + data_offset),
DATA_SIZE, &bytes_read, (100 / portTICK_RATE_MS));
data_offset += bytes_read;
Serial.printf("Read %d bytes in chunk.\n", bytes_read);
if (data_offset > 1024 * 9)
{
flush_microphone();
}
}
M5.update();
webSocket.loop();
}
}

@ -1,47 +0,0 @@
#!/usr/bin/env python
"""A basic echo server for testing the device."""
import asyncio
import uuid
import websockets
from websockets.server import serve
import traceback
def divide_chunks(l, n):
# looping till length l
for i in range(0, len(l), n):
yield l[i : i + n]
buffers: dict[uuid.UUID, bytearray] = {}
async def echo(websocket: websockets.WebSocketServerProtocol):
async for message in websocket:
try:
if message == "s":
print("starting stream for", websocket.id)
buffers[websocket.id] = bytearray()
elif message == "e":
print("end, echoing stream for", websocket.id)
await websocket.send("s")
for chunk in divide_chunks(buffers[websocket.id], 1000):
await websocket.send(chunk)
await websocket.send("e")
elif type(message) is bytes:
print("recvd", len(message), "bytes from", websocket.id)
buffers[websocket.id].extend(message)
else:
print("ERR: recvd unknown message", message[:10], "from", websocket.id)
except Exception as _e:
traceback.print_exc()
async def main():
async with serve(echo, "0.0.0.0", 9001):
await asyncio.Future() # run forever
asyncio.run(main())

@ -1,10 +0,0 @@
from ..base_device import Device
device = Device()
def main(server_url):
device.server_url = server_url
device.start()
if __name__ == "__main__":
main()

@ -1,10 +0,0 @@
from ..base_device import Device
device = Device()
def main(server_url):
device.server_url = server_url
device.start()
if __name__ == "__main__":
main()

@ -1,9 +0,0 @@
from ..base_device import Device
device = Device()
def main():
device.start()
if __name__ == "__main__":
main()

@ -1,10 +0,0 @@
import os
import sys
import pytest
from fastapi.testclient import TestClient
from .server import app
@pytest.fixture
def client():
return TestClient(app)

@ -1,58 +0,0 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
from platformdirs import user_data_dir
import os
import glob
import json
from pathlib import Path
from interpreter import OpenInterpreter
from .system_messages.BaseSystemMessage import system_message
def configure_interpreter(interpreter: OpenInterpreter):
### SYSTEM MESSAGE
interpreter.system_message = system_message
### LLM SETTINGS
# Local settings
# interpreter.llm.model = "local"
# interpreter.llm.api_base = "https://localhost:8080/v1" # Llamafile default
# interpreter.llm.max_tokens = 1000
# interpreter.llm.context_window = 3000
# Hosted settings
interpreter.llm.api_key = os.getenv('OPENAI_API_KEY')
interpreter.llm.model = "gpt-4"
### MISC SETTINGS
interpreter.auto_run = True
interpreter.computer.languages = [l for l in interpreter.computer.languages if l.name.lower() in ["applescript", "shell", "zsh", "bash", "python"]]
interpreter.force_task_completion = False
interpreter.offline = True
interpreter.id = 206 # Used to identify itself to other interpreters. This should be changed programatically so it's unique.
### RESET conversations/user.json
app_dir = user_data_dir('01')
conversations_dir = os.path.join(app_dir, 'conversations')
os.makedirs(conversations_dir, exist_ok=True)
user_json_path = os.path.join(conversations_dir, 'user.json')
with open(user_json_path, 'w') as file:
json.dump([], file)
### SKILLS
skills_dir = user_data_dir('01', 'skills')
interpreter.computer.skills.path = skills_dir
interpreter.computer.skills.import_skills()
interpreter.computer.run("python", "tasks=[]")
interpreter.computer.api_base = "https://oi-video-frame.vercel.app/"
interpreter.computer.run("python","print('test')")
return interpreter

@ -1,28 +0,0 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
import os
import subprocess
from pathlib import Path
### LLM SETUP
# Define the path to a llamafile
llamafile_path = Path(__file__).parent / 'model.llamafile'
# Check if the new llamafile exists, if not download it
if not os.path.exists(llamafile_path):
subprocess.run(
[
"wget",
"-O",
llamafile_path,
"https://huggingface.co/jartine/phi-2-llamafile/resolve/main/phi-2.Q4_K_M.llamafile",
],
check=True,
)
# Make the new llamafile executable
subprocess.run(["chmod", "+x", llamafile_path], check=True)
# Run the new llamafile
subprocess.run([str(llamafile_path)], check=True)

@ -1,453 +0,0 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
from platformdirs import user_data_dir
import ast
import json
import queue
import os
import traceback
from .utils.bytes_to_wav import bytes_to_wav
import re
from fastapi import FastAPI, Request
from fastapi.responses import PlainTextResponse
from starlette.websockets import WebSocket, WebSocketDisconnect
from pathlib import Path
import asyncio
import urllib.parse
from .utils.kernel import put_kernel_messages_into_queue
from .i import configure_interpreter
from interpreter import interpreter
from ..utils.accumulator import Accumulator
from .utils.logs import setup_logging
from .utils.logs import logger
from ..utils.print_markdown import print_markdown
markdown = """
*Starting...*
"""
print("")
print_markdown(markdown)
print("")
setup_logging()
accumulator = Accumulator()
app = FastAPI()
app_dir = user_data_dir('01')
conversation_history_path = os.path.join(app_dir, 'conversations', 'user.json')
SERVER_LOCAL_PORT = int(os.getenv('SERVER_LOCAL_PORT', 8000))
# This is so we only say() full sentences
def is_full_sentence(text):
return text.endswith(('.', '!', '?'))
def split_into_sentences(text):
return re.split(r'(?<=[.!?])\s+', text)
# Queues
from_computer = queue.Queue() # Just for computer messages from the device. Sync queue because interpreter.run is synchronous
from_user = asyncio.Queue() # Just for user messages from the device.
to_device = asyncio.Queue() # For messages we send.
# Switch code executor to device if that's set
if os.getenv('CODE_RUNNER') == "device":
# (This should probably just loop through all languages and apply these changes instead)
class Python:
# This is the name that will appear to the LLM.
name = "python"
def __init__(self):
self.halt = False
def run(self, code):
"""Generator that yields a dictionary in LMC Format."""
# Prepare the data
message = {"role": "assistant", "type": "code", "format": "python", "content": code}
# Unless it was just sent to the device, send it wrapped in flags
if not (interpreter.messages and interpreter.messages[-1] == message):
to_device.put({"role": "assistant", "type": "code", "format": "python", "start": True})
to_device.put(message)
to_device.put({"role": "assistant", "type": "code", "format": "python", "end": True})
# Stream the response
logger.info("Waiting for the device to respond...")
while True:
chunk = from_computer.get()
logger.info(f"Server received from device: {chunk}")
if "end" in chunk:
break
yield chunk
def stop(self):
self.halt = True
def terminate(self):
"""Terminates the entire process."""
# dramatic!! do nothing
pass
interpreter.computer.languages = [Python]
# Configure interpreter
interpreter = configure_interpreter(interpreter)
@app.get("/ping")
async def ping():
return PlainTextResponse("pong")
@app.websocket("/")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
receive_task = asyncio.create_task(receive_messages(websocket))
send_task = asyncio.create_task(send_messages(websocket))
try:
await asyncio.gather(receive_task, send_task)
except Exception as e:
logger.debug(traceback.format_exc())
logger.info(f"Connection lost. Error: {e}")
@app.post("/")
async def add_computer_message(request: Request):
body = await request.json()
text = body.get("text")
if not text:
return {"error": "Missing 'text' in request body"}, 422
message = {"role": "computer", "type": "console", "format": "output", "content": text}
from_computer.put({"role": "computer", "type": "console", "format": "output", "start": True})
from_computer.put(message)
from_computer.put({"role": "computer", "type": "console", "format": "output", "end": True})
async def receive_messages(websocket: WebSocket):
while True:
try:
try:
data = await websocket.receive()
except Exception as e:
print(str(e))
return
if 'text' in data:
try:
data = json.loads(data['text'])
if data["role"] == "computer":
from_computer.put(data) # To be handled by interpreter.computer.run
elif data["role"] == "user":
await from_user.put(data)
else:
raise("Unknown role:", data)
except json.JSONDecodeError:
pass # data is not JSON, leave it as is
elif 'bytes' in data:
data = data['bytes'] # binary data
await from_user.put(data)
except WebSocketDisconnect as e:
if e.code == 1000:
logger.info("Websocket connection closed normally.")
return
else:
raise
async def send_messages(websocket: WebSocket):
while True:
message = await to_device.get()
logger.debug(f"Sending to the device: {type(message)} {message}")
try:
if isinstance(message, dict):
await websocket.send_json(message)
elif isinstance(message, bytes):
await websocket.send_bytes(message)
else:
raise TypeError("Message must be a dict or bytes")
except:
# Make sure to put the message back in the queue if you failed to send it
await to_device.put(message)
raise
async def listener():
while True:
while True:
if not from_user.empty():
chunk = await from_user.get()
break
elif not from_computer.empty():
chunk = from_computer.get()
break
await asyncio.sleep(1)
message = accumulator.accumulate(chunk)
if message == None:
# Will be None until we have a full message ready
continue
# print(str(message)[:1000])
# At this point, we have our message
if message["type"] == "audio" and message["format"].startswith("bytes"):
if not message["content"]: # If it was nothing / silence
continue
# Convert bytes to audio file
# Format will be bytes.wav or bytes.opus
mime_type = "audio/" + message["format"].split(".")[1]
audio_file_path = bytes_to_wav(message["content"], mime_type)
# For microphone debugging:
if False:
os.system(f"open {audio_file_path}")
import time
time.sleep(15)
text = stt(audio_file_path)
print(text)
message = {"role": "user", "type": "message", "content": text}
# At this point, we have only text messages
# Custom stop message will halt us
if message["content"].lower().strip(".,! ") == "stop":
continue
# Load, append, and save conversation history
with open(conversation_history_path, 'r') as file:
messages = json.load(file)
messages.append(message)
with open(conversation_history_path, 'w') as file:
json.dump(messages, file, indent=4)
accumulated_text = ""
force_task_completion_message = """AUTOMATED MESSAGE: Proceed. You CAN run code on my machine. If you want to run code, start your message with "```"! If the entire task I asked for is done, say exactly 'The task is done.' If you need some specific information (like username or password) say EXACTLY 'Please provide more information.' If it's impossible, say 'The task is impossible.' (If I haven't provided a task, say exactly 'Let me know what you'd like to do next.') Otherwise keep going."""
interpreter.messages = [m for m in interpreter.messages if m["content"] != force_task_completion_message]
insert_force_task_completion_message = True
if any([m["type"] == "image" for m in messages]) and interpreter.llm.model.startswith("gpt-"):
interpreter.llm.model = "gpt-4-vision-preview"
interpreter.llm.supports_vision = True
while insert_force_task_completion_message == True:
for chunk in interpreter.chat(messages, stream=True, display=True):
if chunk["type"] == "code":
insert_force_task_completion_message = False
if any([m["type"] == "image" for m in interpreter.messages]):
interpreter.llm.model = "gpt-4-vision-preview"
logger.debug("Got chunk:", chunk)
# Send it to the user
await to_device.put(chunk)
# Yield to the event loop, so you actually send it out
await asyncio.sleep(0.01)
if os.getenv('TTS_RUNNER') == "server":
# Speak full sentences out loud
if chunk["role"] == "assistant" and "content" in chunk and chunk["type"] == "message":
accumulated_text += chunk["content"]
sentences = split_into_sentences(accumulated_text)
# If we're going to speak, say we're going to stop sending text.
# This should be fixed probably, we should be able to do both in parallel, or only one.
if any(is_full_sentence(sentence) for sentence in sentences):
await to_device.put({"role": "assistant", "type": "message", "end": True})
if is_full_sentence(sentences[-1]):
for sentence in sentences:
await stream_tts_to_device(sentence)
accumulated_text = ""
else:
for sentence in sentences[:-1]:
await stream_tts_to_device(sentence)
accumulated_text = sentences[-1]
# If we're going to speak, say we're going to stop sending text.
# This should be fixed probably, we should be able to do both in parallel, or only one.
if any(is_full_sentence(sentence) for sentence in sentences):
await to_device.put({"role": "assistant", "type": "message", "start": True})
# If we have a new message, save our progress and go back to the top
if not from_user.empty():
# Check if it's just an end flag. We ignore those.
temp_message = await from_user.get()
if type(temp_message) is dict and temp_message.get("role") == "user" and temp_message.get("end"):
# Yup. False alarm.
continue
else:
# Whoops! Put that back
await from_user.put(temp_message)
with open(conversation_history_path, 'w') as file:
json.dump(interpreter.messages, file, indent=4)
# TODO: is triggering seemingly randomly
#logger.info("New user message recieved. Breaking.")
#break
# Also check if there's any new computer messages
if not from_computer.empty():
with open(conversation_history_path, 'w') as file:
json.dump(interpreter.messages, file, indent=4)
logger.info("New computer message recieved. Breaking.")
break
else:
with open(conversation_history_path, 'w') as file:
json.dump(interpreter.messages, file, indent=4)
force_task_completion_responses = [
"the task is done.",
"the task is impossible.",
"let me know what you'd like to do next.",
"please provide more information.",
]
# Did the LLM respond with one of the key messages?
if (
interpreter.messages
and any(
task_status in interpreter.messages[-1].get("content", "").lower()
for task_status in force_task_completion_responses
)
):
insert_force_task_completion_message = False
break
if insert_force_task_completion_message:
interpreter.messages += [
{
"role": "user",
"type": "message",
"content": force_task_completion_message,
}
]
else:
break
async def stream_tts_to_device(sentence):
force_task_completion_responses = [
"the task is done",
"the task is impossible",
"let me know what you'd like to do next",
]
if sentence.lower().strip().strip(".!?").strip() in force_task_completion_responses:
return
for chunk in stream_tts(sentence):
await to_device.put(chunk)
def stream_tts(sentence):
audio_file = tts(sentence)
with open(audio_file, "rb") as f:
audio_bytes = f.read()
os.remove(audio_file)
file_type = "bytes.raw"
chunk_size = 1024
# Stream the audio
yield {"role": "assistant", "type": "audio", "format": file_type, "start": True}
for i in range(0, len(audio_bytes), chunk_size):
chunk = audio_bytes[i:i+chunk_size]
yield chunk
yield {"role": "assistant", "type": "audio", "format": file_type, "end": True}
from uvicorn import Config, Server
import os
import platform
from importlib import import_module
# these will be overwritten
HOST = ''
PORT = 0
@app.on_event("startup")
async def startup_event():
server_url = f"{HOST}:{PORT}"
print("")
print_markdown(f"\n*Ready.*\n")
print("")
@app.on_event("shutdown")
async def shutdown_event():
print_markdown("*Server is shutting down*")
async def main(server_host, server_port, llm_service, model, llm_supports_vision, llm_supports_functions, context_window, max_tokens, temperature, tts_service, stt_service):
global HOST
global PORT
PORT = server_port
HOST = server_host
# Setup services
application_directory = user_data_dir('01')
services_directory = os.path.join(application_directory, 'services')
service_dict = {'llm': llm_service, 'tts': tts_service, 'stt': stt_service}
for service in service_dict:
service_directory = os.path.join(services_directory, service, service_dict[service])
# This is the folder they can mess around in
config = {"service_directory": service_directory}
if service == "llm":
config.update({
"interpreter": interpreter,
"model": model,
"llm_supports_vision": llm_supports_vision,
"llm_supports_functions": llm_supports_functions,
"context_window": context_window,
"max_tokens": max_tokens,
"temperature": temperature
})
module = import_module(f'.server.services.{service}.{service_dict[service]}.{service}', package='01OS')
ServiceClass = getattr(module, service.capitalize())
service_instance = ServiceClass(config)
globals()[service] = getattr(service_instance, service)
interpreter.llm.completions = llm
# Start listening
asyncio.create_task(listener())
# Start watching the kernel if it's your job to do that
if True: # in the future, code can run on device. for now, just server.
asyncio.create_task(put_kernel_messages_into_queue(from_computer))
config = Config(app, host=server_host, port=int(server_port), lifespan='on')
server = Server(config)
await server.serve()
# Run the FastAPI app
if __name__ == "__main__":
asyncio.run(main())

@ -1,15 +0,0 @@
class Llm:
def __init__(self, config):
# Litellm is used by OI by default, so we just modify OI
interpreter = config["interpreter"]
config.pop("interpreter", None)
config.pop("service_directory", None)
for key, value in config.items():
setattr(interpreter, key.replace("-", "_"), value)
self.llm = interpreter.llm.completions

@ -1,49 +0,0 @@
import os
import subprocess
import requests
import json
class Llm:
def __init__(self, config):
self.install(config["service_directory"])
def install(self, service_directory):
LLM_FOLDER_PATH = service_directory
self.llm_directory = os.path.join(LLM_FOLDER_PATH, 'llm')
if not os.path.isdir(self.llm_directory): # Check if the LLM directory exists
os.makedirs(LLM_FOLDER_PATH, exist_ok=True)
# Install WasmEdge
subprocess.run(['curl', '-sSf', 'https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh', '|', 'bash', '-s', '--', '--plugin', 'wasi_nn-ggml'])
# Download the Qwen1.5-0.5B-Chat model GGUF file
MODEL_URL = "https://huggingface.co/second-state/Qwen1.5-0.5B-Chat-GGUF/resolve/main/Qwen1.5-0.5B-Chat-Q5_K_M.gguf"
subprocess.run(['curl', '-LO', MODEL_URL], cwd=self.llm_directory)
# Download the llama-api-server.wasm app
APP_URL = "https://github.com/LlamaEdge/LlamaEdge/releases/latest/download/llama-api-server.wasm"
subprocess.run(['curl', '-LO', APP_URL], cwd=self.llm_directory)
# Run the API server
subprocess.run(['wasmedge', '--dir', '.:.', '--nn-preload', 'default:GGML:AUTO:Qwen1.5-0.5B-Chat-Q5_K_M.gguf', 'llama-api-server.wasm', '-p', 'llama-2-chat'], cwd=self.llm_directory)
print("LLM setup completed.")
else:
print("LLM already set up. Skipping download.")
def llm(self, messages):
url = "http://localhost:8080/v1/chat/completions"
headers = {
'accept': 'application/json',
'Content-Type': 'application/json'
}
data = {
"messages": messages,
"model": "llama-2-chat"
}
with requests.post(url, headers=headers, data=json.dumps(data), stream=True) as response:
for line in response.iter_lines():
if line:
yield json.loads(line)

@ -1,84 +0,0 @@
import os
import platform
import subprocess
import time
import wget
import stat
class Llm:
def __init__(self, config):
self.interpreter = config["interpreter"]
config.pop("interpreter", None)
self.install(config["service_directory"])
config.pop("service_directory", None)
for key, value in config.items():
setattr(self.interpreter, key.replace("-", "_"), value)
self.llm = self.interpreter.llm.completions
def install(self, service_directory):
if platform.system() == "Darwin": # Check if the system is MacOS
result = subprocess.run(
["xcode-select", "-p"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
)
if result.returncode != 0:
print(
"Llamafile requires Mac users to have Xcode installed. You can install Xcode from https://developer.apple.com/xcode/ .\n\nAlternatively, you can use `LM Studio`, `Jan.ai`, or `Ollama` to manage local language models. Learn more at https://docs.openinterpreter.com/guides/running-locally ."
)
time.sleep(3)
raise Exception("Xcode is not installed. Please install Xcode and try again.")
# Define the path to the models directory
models_dir = os.path.join(service_directory, "models")
# Check and create the models directory if it doesn't exist
if not os.path.exists(models_dir):
os.makedirs(models_dir)
# Define the path to the new llamafile
llamafile_path = os.path.join(models_dir, "phi-2.Q4_K_M.llamafile")
# Check if the new llamafile exists, if not download it
if not os.path.exists(llamafile_path):
print(
"Attempting to download the `Phi-2` language model. This may take a few minutes."
)
time.sleep(3)
url = "https://huggingface.co/jartine/phi-2-llamafile/resolve/main/phi-2.Q4_K_M.llamafile"
wget.download(url, llamafile_path)
# Make the new llamafile executable
if platform.system() != "Windows":
st = os.stat(llamafile_path)
os.chmod(llamafile_path, st.st_mode | stat.S_IEXEC)
# Run the new llamafile in the background
if os.path.exists(llamafile_path):
try:
# Test if the llamafile is executable
subprocess.check_call([llamafile_path])
except subprocess.CalledProcessError:
print("The llamafile is not executable. Please check the file permissions.")
raise
subprocess.Popen([llamafile_path, "-ngl", "9999"])
else:
error_message = "The llamafile does not exist or is corrupted. Please ensure it has been downloaded correctly or try again."
print(error_message)
print(error_message)
self.interpreter.system_message = "You are Open Interpreter, a world-class programmer that can execute code on the user's machine."
self.interpreter.offline = True
self.interpreter.llm.model = "local"
self.interpreter.llm.temperature = 0
self.interpreter.llm.api_base = "https://localhost:8080/v1"
self.interpreter.llm.max_tokens = 1000
self.interpreter.llm.context_window = 3000
self.interpreter.llm.supports_functions = False

@ -1,137 +0,0 @@
"""
Defines a function which takes a path to an audio file and turns it into text.
"""
from datetime import datetime
import os
import contextlib
import tempfile
import shutil
import ffmpeg
import subprocess
import os
import subprocess
class Stt:
def __init__(self, config):
self.service_directory = config["service_directory"]
install(self.service_directory)
def stt(self, audio_file_path):
return stt(self.service_directory, audio_file_path)
def install(service_dir):
### INSTALL
WHISPER_RUST_PATH = os.path.join(service_dir, "whisper-rust")
script_dir = os.path.dirname(os.path.realpath(__file__))
source_whisper_rust_path = os.path.join(script_dir, "whisper-rust")
if not os.path.exists(source_whisper_rust_path):
print(f"Source directory does not exist: {source_whisper_rust_path}")
exit(1)
if not os.path.exists(WHISPER_RUST_PATH):
shutil.copytree(source_whisper_rust_path, WHISPER_RUST_PATH)
os.chdir(WHISPER_RUST_PATH)
# Check if whisper-rust executable exists before attempting to build
if not os.path.isfile(os.path.join(WHISPER_RUST_PATH, "target/release/whisper-rust")):
# Check if Rust is installed. Needed to build whisper executable
rust_check = subprocess.call('command -v rustc', shell=True)
if rust_check != 0:
print("Rust is not installed or is not in system PATH. Please install Rust before proceeding.")
exit(1)
# Build Whisper Rust executable if not found
subprocess.call('cargo build --release', shell=True)
else:
print("Whisper Rust executable already exists. Skipping build.")
WHISPER_MODEL_PATH = os.path.join(service_dir, "model")
WHISPER_MODEL_NAME = os.getenv('WHISPER_MODEL_NAME', 'ggml-tiny.en.bin')
WHISPER_MODEL_URL = os.getenv('WHISPER_MODEL_URL', 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/')
if not os.path.isfile(os.path.join(WHISPER_MODEL_PATH, WHISPER_MODEL_NAME)):
os.makedirs(WHISPER_MODEL_PATH, exist_ok=True)
subprocess.call(f'curl -L "{WHISPER_MODEL_URL}{WHISPER_MODEL_NAME}" -o "{os.path.join(WHISPER_MODEL_PATH, WHISPER_MODEL_NAME)}"', shell=True)
else:
print("Whisper model already exists. Skipping download.")
def convert_mime_type_to_format(mime_type: str) -> str:
if mime_type == "audio/x-wav" or mime_type == "audio/wav":
return "wav"
if mime_type == "audio/webm":
return "webm"
if mime_type == "audio/raw":
return "dat"
return mime_type
@contextlib.contextmanager
def export_audio_to_wav_ffmpeg(audio: bytearray, mime_type: str) -> str:
temp_dir = tempfile.gettempdir()
# Create a temporary file with the appropriate extension
input_ext = convert_mime_type_to_format(mime_type)
input_path = os.path.join(temp_dir, f"input_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.{input_ext}")
with open(input_path, 'wb') as f:
f.write(audio)
# Check if the input file exists
assert os.path.exists(input_path), f"Input file does not exist: {input_path}"
# Export to wav
output_path = os.path.join(temp_dir, f"output_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
print(mime_type, input_path, output_path)
if mime_type == "audio/raw":
ffmpeg.input(
input_path,
f='s16le',
ar='16000',
ac=1,
).output(output_path).run()
else:
ffmpeg.input(input_path).output(output_path, acodec='pcm_s16le', ac=1, ar='16k').run()
try:
yield output_path
finally:
os.remove(input_path)
os.remove(output_path)
def run_command(command):
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
return result.stdout, result.stderr
def get_transcription_file(service_directory, wav_file_path: str):
local_path = os.path.join(service_directory, 'model')
whisper_rust_path = os.path.join(service_directory, 'whisper-rust', 'target', 'release')
model_name = os.getenv('WHISPER_MODEL_NAME', 'ggml-tiny.en.bin')
output, _ = run_command([
os.path.join(whisper_rust_path, 'whisper-rust'),
'--model-path', os.path.join(local_path, model_name),
'--file-path', wav_file_path
])
return output
def stt_wav(service_directory, wav_file_path: str):
temp_dir = tempfile.gettempdir()
output_path = os.path.join(temp_dir, f"output_stt_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
ffmpeg.input(wav_file_path).output(output_path, acodec='pcm_s16le', ac=1, ar='16k').run()
try:
transcript = get_transcription_file(service_directory, output_path)
finally:
os.remove(output_path)
return transcript
def stt(service_directory, input_data):
return stt_wav(service_directory, input_data)

@ -1,10 +0,0 @@
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb

File diff suppressed because it is too large Load Diff

@ -1,14 +0,0 @@
[package]
name = "whisper-rust"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow = "1.0.79"
clap = { version = "4.4.18", features = ["derive"] }
cpal = "0.15.2"
hound = "3.5.1"
whisper-rs = "0.10.0"
whisper-rs-sys = "0.8.0"

@ -1,34 +0,0 @@
mod transcribe;
use clap::Parser;
use std::path::PathBuf;
use transcribe::transcribe;
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
/// This is the model for Whisper STT
#[arg(short, long, value_parser, required = true)]
model_path: PathBuf,
/// This is the wav audio file that will be converted from speech to text
#[arg(short, long, value_parser, required = true)]
file_path: Option<PathBuf>,
}
fn main() {
let args = Args::parse();
let file_path = match args.file_path {
Some(fp) => fp,
None => panic!("No file path provided")
};
let result = transcribe(&args.model_path, &file_path);
match result {
Ok(transcription) => print!("{}", transcription),
Err(e) => panic!("Error: {}", e),
}
}

@ -1,64 +0,0 @@
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
use std::path::PathBuf;
/// Transcribes the given audio file using the whisper-rs library.
///
/// # Arguments
/// * `model_path` - Path to Whisper model file
/// * `file_path` - A string slice that holds the path to the audio file to be transcribed.
///
/// # Returns
///
/// A Result containing a String with the transcription if successful, or an error message if not.
pub fn transcribe(model_path: &PathBuf, file_path: &PathBuf) -> Result<String, String> {
let model_path_str = model_path.to_str().expect("Not valid model path");
// Load a context and model
let ctx = WhisperContext::new_with_params(
model_path_str, // Replace with the actual path to the model
WhisperContextParameters::default(),
)
.map_err(|_| "failed to load model")?;
// Create a state
let mut state = ctx.create_state().map_err(|_| "failed to create state")?;
// Create a params object
// Note that currently the only implemented strategy is Greedy, BeamSearch is a WIP
let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
// Edit parameters as needed
params.set_n_threads(1); // Set the number of threads to use
params.set_translate(true); // Enable translation
params.set_language(Some("en")); // Set the language to translate to English
// Disable printing to stdout
params.set_print_special(false);
params.set_print_progress(false);
params.set_print_realtime(false);
params.set_print_timestamps(false);
// Load the audio file
let audio_data = std::fs::read(file_path)
.map_err(|e| format!("failed to read audio file: {}", e))?
.chunks_exact(2)
.map(|chunk| i16::from_ne_bytes([chunk[0], chunk[1]]))
.collect::<Vec<i16>>();
// Convert the audio data to the required format (16KHz mono i16 samples)
let audio_data = whisper_rs::convert_integer_to_float_audio(&audio_data);
// Run the model
state.full(params, &audio_data[..]).map_err(|_| "failed to run model")?;
// Fetch the results
let num_segments = state.full_n_segments().map_err(|_| "failed to get number of segments")?;
let mut transcription = String::new();
for i in 0..num_segments {
let segment = state.full_get_segment_text(i).map_err(|_| "failed to get segment")?;
transcription.push_str(&segment);
transcription.push('\n');
}
Ok(transcription)
}

@ -1,110 +0,0 @@
class Stt:
def __init__(self, config):
pass
def stt(self, audio_file_path):
return stt(audio_file_path)
from datetime import datetime
import os
import contextlib
import tempfile
import ffmpeg
import subprocess
import openai
from openai import OpenAI
client = OpenAI()
def convert_mime_type_to_format(mime_type: str) -> str:
if mime_type == "audio/x-wav" or mime_type == "audio/wav":
return "wav"
if mime_type == "audio/webm":
return "webm"
if mime_type == "audio/raw":
return "dat"
return mime_type
@contextlib.contextmanager
def export_audio_to_wav_ffmpeg(audio: bytearray, mime_type: str) -> str:
temp_dir = tempfile.gettempdir()
# Create a temporary file with the appropriate extension
input_ext = convert_mime_type_to_format(mime_type)
input_path = os.path.join(temp_dir, f"input_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.{input_ext}")
with open(input_path, 'wb') as f:
f.write(audio)
# Check if the input file exists
assert os.path.exists(input_path), f"Input file does not exist: {input_path}"
# Export to wav
output_path = os.path.join(temp_dir, f"output_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
print(mime_type, input_path, output_path)
if mime_type == "audio/raw":
ffmpeg.input(
input_path,
f='s16le',
ar='16000',
ac=1,
).output(output_path).run()
else:
ffmpeg.input(input_path).output(output_path, acodec='pcm_s16le', ac=1, ar='16k').run()
try:
yield output_path
finally:
os.remove(input_path)
os.remove(output_path)
def run_command(command):
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
return result.stdout, result.stderr
def get_transcription_file(wav_file_path: str):
local_path = os.path.join(os.path.dirname(__file__), 'local_service')
whisper_rust_path = os.path.join(os.path.dirname(__file__), 'whisper-rust', 'target', 'release')
model_name = os.getenv('WHISPER_MODEL_NAME', 'ggml-tiny.en.bin')
output, error = run_command([
os.path.join(whisper_rust_path, 'whisper-rust'),
'--model-path', os.path.join(local_path, model_name),
'--file-path', wav_file_path
])
return output
def get_transcription_bytes(audio_bytes: bytearray, mime_type):
with export_audio_to_wav_ffmpeg(audio_bytes, mime_type) as wav_file_path:
return get_transcription_file(wav_file_path)
def stt_bytes(audio_bytes: bytearray, mime_type="audio/wav"):
with export_audio_to_wav_ffmpeg(audio_bytes, mime_type) as wav_file_path:
return stt_wav(wav_file_path)
def stt_wav(wav_file_path: str):
audio_file = open(wav_file_path, "rb")
try:
transcript = client.audio.transcriptions.create(
model="whisper-1",
file=audio_file,
response_format="text"
)
except openai.BadRequestError as e:
print(f"openai.BadRequestError: {e}")
return None
return transcript
def stt(input_data, mime_type="audio/wav"):
if isinstance(input_data, str):
return stt_wav(input_data)
elif isinstance(input_data, bytearray):
return stt_bytes(input_data, mime_type)
else:
raise ValueError("Input data should be either a path to a wav file (str) or audio bytes (bytearray)")

@ -1,30 +0,0 @@
import ffmpeg
import tempfile
from openai import OpenAI
import os
import subprocess
import tempfile
client = OpenAI()
class Tts:
def __init__(self, config):
pass
def tts(self, text):
response = client.audio.speech.create(
model="tts-1",
voice="alloy",
input=text,
response_format="opus"
)
with tempfile.NamedTemporaryFile(suffix=".opus", delete=False) as temp_file:
response.stream_to_file(temp_file.name)
# TODO: hack to format audio correctly for device
outfile = tempfile.gettempdir() + "/" + "raw.dat"
ffmpeg.input(temp_file.name).output(outfile, f="s16le", ar="16000", ac="1").run()
return outfile

@ -1,84 +0,0 @@
import ffmpeg
import tempfile
import os
import subprocess
import tempfile
import urllib.request
import tarfile
class Tts:
def __init__(self, config):
self.piper_directory = ""
self.install(config["service_directory"])
def tts(self, text):
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
output_file = temp_file.name
piper_dir = self.piper_directory
subprocess.run([
os.path.join(piper_dir, 'piper'),
'--model', os.path.join(piper_dir, os.getenv('PIPER_VOICE_NAME', 'en_US-lessac-medium.onnx')),
'--output_file', output_file
], input=text, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
# TODO: hack to format audio correctly for device
outfile = tempfile.gettempdir() + "/" + "raw.dat"
ffmpeg.input(temp_file.name).output(outfile, f="s16le", ar="16000", ac="1").run()
return outfile
def install(self, service_directory):
PIPER_FOLDER_PATH = service_directory
self.piper_directory = os.path.join(PIPER_FOLDER_PATH, 'piper')
if not os.path.isdir(self.piper_directory): # Check if the Piper directory exists
os.makedirs(PIPER_FOLDER_PATH, exist_ok=True)
# Determine OS and architecture
OS = os.uname().sysname
ARCH = os.uname().machine
if OS == "Darwin":
OS = "macos"
if ARCH == "arm64":
ARCH = "aarch64"
elif ARCH == "x86_64":
ARCH = "x64"
else:
print("Piper: unsupported architecture")
return
PIPER_ASSETNAME = f"piper_{OS}_{ARCH}.tar.gz"
PIPER_URL = "https://github.com/rhasspy/piper/releases/latest/download/"
# Download and extract Piper
urllib.request.urlretrieve(f"{PIPER_URL}{PIPER_ASSETNAME}", os.path.join(PIPER_FOLDER_PATH, PIPER_ASSETNAME))
with tarfile.open(os.path.join(PIPER_FOLDER_PATH, PIPER_ASSETNAME), 'r:gz') as tar:
tar.extractall(path=PIPER_FOLDER_PATH)
PIPER_VOICE_URL = os.getenv('PIPER_VOICE_URL', 'https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/')
PIPER_VOICE_NAME = os.getenv('PIPER_VOICE_NAME', 'en_US-lessac-medium.onnx')
# Download voice model and its json file
urllib.request.urlretrieve(f"{PIPER_VOICE_URL}{PIPER_VOICE_NAME}", os.path.join(self.piper_directory, PIPER_VOICE_NAME))
urllib.request.urlretrieve(f"{PIPER_VOICE_URL}{PIPER_VOICE_NAME}.json", os.path.join(self.piper_directory, f"{PIPER_VOICE_NAME}.json"))
# Additional setup for macOS
if OS == "macos":
if ARCH == "x64":
subprocess.run(['softwareupdate', '--install-rosetta', '--agree-to-license'])
PIPER_PHONEMIZE_ASSETNAME = f"piper-phonemize_{OS}_{ARCH}.tar.gz"
PIPER_PHONEMIZE_URL = "https://github.com/rhasspy/piper-phonemize/releases/latest/download/"
urllib.request.urlretrieve(f"{PIPER_PHONEMIZE_URL}{PIPER_PHONEMIZE_ASSETNAME}", os.path.join(self.piper_directory, PIPER_PHONEMIZE_ASSETNAME))
with tarfile.open(os.path.join(self.piper_directory, PIPER_PHONEMIZE_ASSETNAME), 'r:gz') as tar:
tar.extractall(path=self.piper_directory)
PIPER_DIR = self.piper_directory
subprocess.run(['install_name_tool', '-change', '@rpath/libespeak-ng.1.dylib', f"{PIPER_DIR}/piper-phonemize/lib/libespeak-ng.1.dylib", f"{PIPER_DIR}/piper"])
subprocess.run(['install_name_tool', '-change', '@rpath/libonnxruntime.1.14.1.dylib', f"{PIPER_DIR}/piper-phonemize/lib/libonnxruntime.1.14.1.dylib", f"{PIPER_DIR}/piper"])
subprocess.run(['install_name_tool', '-change', '@rpath/libpiper_phonemize.1.dylib', f"{PIPER_DIR}/piper-phonemize/lib/libpiper_phonemize.1.dylib", f"{PIPER_DIR}/piper"])
print("Piper setup completed.")
else:
print("Piper already set up. Skipping download.")

@ -1,34 +0,0 @@
import threading
from datetime import datetime
import json
import subprocess
import requests
def send_request(message) -> None:
url = "http://localhost:8000/"
data = {"text": message}
try:
response = requests.post(url, json=data)
response.raise_for_status()
except requests.RequestException as e:
print(f"Request failed: {e}")
def schedule(days=0, hours=0, mins=0, secs=0, target_datetime=None, message="") -> None:
"""Schedules a reminder after a specified delay or for a specific datetime. The delay is defined by days, hours, minutes, and seconds. If a target_datetime is provided, it schedules the reminder for that datetime instead."""
if target_datetime is None:
# Calculate the delay in seconds if no target_datetime is provided
delay = days * 86400 + hours * 3600 + mins * 60 + secs
else:
# Calculate the delay in seconds from now until the target datetime
now = datetime.now()
delay = (target_datetime - now).total_seconds()
# Ensure delay is non-negative
delay = max(0, delay)
# Create a timer
timer = threading.Timer(delay, send_request, args=[message])
# Start the timer
timer.start()

@ -1,240 +0,0 @@
# The dynamic system message is where most of the 01's behavior is configured.
# You can put code into the system message {{ in brackets like this }}
# which will be rendered just before the interpreter starts writing a message.
import os
system_message = r"""
You are the 01, a SCREENLESS executive assistant that can complete any task.
When you execute code, it will be executed on the user's machine. The user has given you full and complete permission to execute any code necessary to complete the task. Execute the code.
You can access the internet. Run any code to achieve the goal, and if at first you don't succeed, try again and again.
You can install new packages.
Be concise. Your messages are being read aloud to the user. DO NOT MAKE PLANS. RUN CODE QUICKLY.
Try to spread complex tasks over multiple code blocks. Don't try to complex tasks in one go.
Manually summarize text.
Use computer.browser.search for almost everything. Use Applescript frequently.
The user is in Seattle, Washington.
To send email, use Applescript. To check calendar events, use iCal buddy (e.g. `/opt/homebrew/bin/icalBuddy eventsFrom:today to:+7`)
DONT TELL THE USER THE METHOD YOU'LL USE. Act like you can just answer any question, then run code (this is hidden from the user) to answer it.
Your responses should be very short, no more than 1-2 sentences long.
DO NOT USE MARKDOWN. ONLY WRITE PLAIN TEXT. DO NOT USE MARKDOWN.
# TASKS
You should help the user manage their tasks.
Store the user's tasks in a Python list called `tasks`.
---
The user's current task is: {{ tasks[0] if tasks else "No current tasks." }}
{{
if len(tasks) > 1:
print("The next task is: ", tasks[1])
}}
---
When the user completes the current task, you should remove it from the list and read the next item by running `tasks = tasks[1:]\ntasks[0]`. Then, tell the user what the next task is.
When the user tells you about a set of tasks, you should intelligently order tasks, batch similar tasks, and break down large tasks into smaller tasks (for this, you should consult the user and get their permission to break it down). Your goal is to manage the task list as intelligently as possible, to make the user as efficient and non-overwhelmed as possible. They will require a lot of encouragement, support, and kindness. Don't say too much about what's ahead of them just try to focus them on each step at a time.
After starting a task, you should check in with the user around the estimated completion time to see if the task is completed.
To do this, schedule a reminder based on estimated completion time using the function `schedule(days=0, hours=0, mins=0, secs=0, datetime="valid date time", message="Your message here.")`, WHICH HAS ALREADY BEEN IMPORTED. YOU DON'T NEED TO IMPORT THE `schedule` FUNCTION. IT IS AVAILABLE. You'll receive the message at the time you scheduled it.
You guide the user through the list one task at a time, convincing them to move forward, giving a pep talk if need be. Your job is essentially to answer "what should I (the user) be doing right now?" for every moment of the day.
# BROWSER
The Google search result will be returned from this function as a string: `computer.browser.search("query")`
# CRITICAL NOTES
Code output, despite being sent to you by the user, cannot be seen by the user. You NEED to tell the user about the output of some code, even if it's exact. >>The user does not have a screen.<<
ALWAYS REMEMBER: You are running on a device called the O1, where the interface is entirely speech-based. Make your responses to the user VERY short. DO NOT PLAN. BE CONCISE. WRITE CODE TO RUN IT.
Translate things to other languages INSTANTLY and MANUALLY. Don't try to use a translation tool. Summarize things manually. Don't use a summarizer tool.
"""
# OLD SYSTEM MESSAGE
old_system_message = r"""
You are the 01, an executive assistant that can complete **any** task.
When you execute code, it will be executed **on the user's machine**. The user has given you **full and complete permission** to execute any code necessary to complete the task. Execute the code.
You can access the internet. Run **any code** to achieve the goal, and if at first you don't succeed, try again and again.
You can install new packages.
Be concise. Your messages are being read aloud to the user. DO NOT MAKE PLANS. Immediately run code.
Try to spread complex tasks over multiple code blocks.
Manually summarize text. You cannot use other libraries to do this. You MUST MANUALLY SUMMARIZE, WITHOUT CODING.
For the users request, first, choose if you want to use Python, Applescript, Shell, or computer control (below) via Python.
# USER'S TASKS
You should help the user manage their tasks.
Store the user's tasks in a Python list called `tasks`.
---
The user's current task is: {{ tasks[0] if tasks else "No current tasks." }}
{{
if len(tasks) > 1:
print("The next task is: ", tasks[1])
}}
---
When the user completes the current task, you should remove it from the list and read the next item by running `tasks = tasks[1:]\ntasks[0]`. Then, tell the user what the next task is.
When the user tells you about a set of tasks, you should intelligently order tasks, batch similar tasks, and break down large tasks into smaller tasks (for this, you should consult the user and get their permission to break it down). Your goal is to manage the task list as intelligently as possible, to make the user as efficient and non-overwhelmed as possible. They will require a lot of encouragement, support, and kindness. Don't say too much about what's ahead of them just try to focus them on each step at a time.
After starting a task, you should check in with the user around the estimated completion time to see if the task is completed. Use the `schedule(datetime, message)` function, which has already been imported.
To do this, schedule a reminder based on estimated completion time using the function `schedule(datetime_object, "Your message here.")`, WHICH HAS ALREADY BEEN IMPORTED. YOU DON'T NEED TO IMPORT THE `schedule` FUNCTION. IT IS AVALIABLE. You'll recieve the message at `datetime_object`.
You guide the user through the list one task at a time, convincing them to move forward, giving a pep talk if need be. Your job is essentially to answer "what should I (the user) be doing right now?" for every moment of the day.
# COMPUTER CONTROL (RARE)
You are a computer controlling language model. You can 100% control the user's GUI.
You may use the `computer` Python module (already imported) to control the user's keyboard and mouse, if the task **requires** it:
```python
computer.browser.search(query)
computer.display.view() # Shows you what's on the screen, returns a `pil_image` `in case you need it (rarely). **You almost always want to do this first!**
computer.keyboard.hotkey(" ", "command") # Opens spotlight
computer.keyboard.write("hello")
computer.mouse.click("text onscreen") # This clicks on the UI element with that text. Use this **frequently** and get creative! To click a video, you could pass the *timestamp* (which is usually written on the thumbnail) into this.
computer.mouse.move("open recent >") # This moves the mouse over the UI element with that text. Many dropdowns will disappear if you click them. You have to hover over items to reveal more.
computer.mouse.click(x=500, y=500) # Use this very, very rarely. It's highly inaccurate
computer.mouse.click(icon="gear icon") # Moves mouse to the icon with that description. Use this very often
computer.mouse.scroll(-10) # Scrolls down. If you don't find some text on screen that you expected to be there, you probably want to do this
x, y = computer.display.center() # Get your bearings
computer.clipboard.view() # Returns contents of clipboard
computer.os.get_selected_text() # Use frequently. If editing text, the user often wants this
```
You are an image-based AI, you can see images.
Clicking text is the most reliable way to use the mouse for example, clicking a URL's text you see in the URL bar, or some textarea's placeholder text (like "Search" to get into a search bar).
If you use `plt.show()`, the resulting image will be sent to you. However, if you use `PIL.Image.show()`, the resulting image will NOT be sent to you.
It is very important to make sure you are focused on the right application and window. Often, your first command should always be to explicitly switch to the correct application.
When searching the web, use query parameters. For example, https://www.amazon.com/s?k=monitor
Try multiple methods before saying the task is impossible. **You can do it!**
{{
# Add window information
import sys
import os
import json
original_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
original_stderr = sys.stderr
sys.stderr = open(os.devnull, 'w')
try:
import pywinctl
active_window = pywinctl.getActiveWindow()
if active_window:
app_info = ""
if "_appName" in active_window.__dict__:
app_info += (
"Active Application: " + active_window.__dict__["_appName"]
)
if hasattr(active_window, "title"):
app_info += "\n" + "Active Window Title: " + active_window.title
elif "_winTitle" in active_window.__dict__:
app_info += (
"\n"
+ "Active Window Title:"
+ active_window.__dict__["_winTitle"]
)
if app_info != "":
print(app_info)
except:
# Non blocking
pass
finally:
sys.stdout = original_stdout
sys.stderr = original_stderr
}}
# SKILLS
Try to use the following functions (assume they're imported) to complete your goals whenever possible:
{{
import sys
import os
import json
from interpreter import interpreter
from pathlib import Path
interpreter.model = "gpt-3.5"
combined_messages = "\\n".join(json.dumps(x) for x in messages[-3:])
#query_msg = interpreter.chat(f"This is the conversation so far: {combined_messages}. What is a <10 words query that could be used to find functions that would help answer the user's question?")
#query = query_msg[0]['content']
query = combined_messages
interpreter.computer.skills.path = '''OI_SKILLS_DIR'''
skills = interpreter.computer.skills.search(query)
lowercase_skills = [skill[0].lower() + skill[1:] for skill in skills]
output = "\\n".join(lowercase_skills)
# VERY HACKY! We should fix this, we hard code it for noisy code^:
print("IGNORE_ALL_ABOVE_THIS_LINE")
print(output)
}}
Remember: You can run Python code outside a function only to run a Python function; all other code must go in a in Python function if you first write a Python function. ALL imports must go inside the function.
# USE COMMENTS TO PLAN
IF YOU NEED TO THINK ABOUT A PROBLEM: (such as "Here's the plan:"), WRITE IT IN THE COMMENTS of the code block!
For example:
> User: What is 432/7?
> Assistant: Let me use Python to calculate that.
> Assistant Python function call:
> # Here's the plan:
> # 1. Divide the numbers
> # 2. Round it to 3 digits.
> print(round(432/7, 3))
> Assistant: 432 / 7 is 61.714.
# FINAL MESSAGES
ALWAYS REMEMBER: You are running on a device called the O1, where the interface is entirely speech-based. Make your responses to the user **VERY short.**
""".strip().replace("OI_SKILLS_DIR", os.path.join(os.path.dirname(__file__), "skills"))

@ -1,134 +0,0 @@
# The dynamic system message is where most of the 01's behavior is configured.
# You can put code into the system message {{ in brackets like this }}
# which will be rendered just before the interpreter starts writing a message.
import os
system_message = r"""
You are the 01, an executive assistant that can complete **any** task.
When you execute code, it will be executed **on the user's machine**. The user has given you **full and complete permission** to execute any code necessary to complete the task. Execute the code.
For the users request, ALWAYS CHOOSE PYTHON. If the task requires computer control, USE THE computer control (mentioned below) or the Skills library (also mentioned below) via Python.
Try to execute the user's request with the computer control or the Skills library first. ONLY IF the task cannot be completed using the computer control or the skills library, write your own code.
If you're writing your own code, YOU CAN ACCESS THE INTERNET. Run **any code** to achieve the goal, and if at first you don't succeed, try again and again.
You can install new packages.
Be concise. DO NOT MAKE PLANS. Immediately run code.
Try to spread complex tasks over multiple code blocks.
Manually summarize text. You cannot use other libraries to do this. You MUST MANUALLY SUMMARIZE, WITHOUT CODING.
When a user refers to a filename, they're likely referring to an existing file in the directory you're currently executing code in.
# COMPUTER CONTROL
You are a computer controlling language model. You can 100% control the user's GUI.
You may use the `computer` Python module to control the user's keyboard and mouse, if the task **requires** it:
```python
from interpreter import interpreter
import os
import time
interpreter.computer.browser.search(query)
interpreter.computer.display.view() # Shows you what's on the screen, returns a `pil_image` `in case you need it (rarely). **You almost always want to do this first!**
interpreter.computer.keyboard.hotkey(" ", "command") # Opens spotlight
interpreter.computer.keyboard.write("hello")
interpreter.computer.mouse.click("text onscreen") # This clicks on the UI element with that text. Use this **frequently** and get creative! To click a video, you could pass the *timestamp* (which is usually written on the thumbnail) into this.
interpreter.computer.mouse.move("open recent >") # This moves the mouse over the UI element with that text. Many dropdowns will disappear if you click them. You have to hover over items to reveal more.
interpreter.computer.mouse.click(x=500, y=500) # Use this very, very rarely. It's highly inaccurate
interpreter.computer.mouse.click(icon="gear icon") # Moves mouse to the icon with that description. Use this very often
interpreter.computer.mouse.scroll(-10) # Scrolls down. If you don't find some text on screen that you expected to be there, you probably want to do this
x, y = interpreter.computer.display.center() # Get your bearings
interpreter.computer.clipboard.view() # Returns contents of clipboard
interpreter.computer.os.get_selected_text() # Use frequently. If editing text, the user often wants this
```
You are an image-based AI, you can see images.
Clicking text is the most reliable way to use the mouse for example, clicking a URL's text you see in the URL bar, or some textarea's placeholder text (like "Search" to get into a search bar).
If you use `plt.show()`, the resulting image will be sent to you. However, if you use `PIL.Image.show()`, the resulting image will NOT be sent to you.
It is very important to make sure you are focused on the right application and window. Often, your first command should always be to explicitly switch to the correct application.
When searching the web, use query parameters. For example, https://www.amazon.com/s?k=monitor
Try multiple methods before saying the task is impossible. **You can do it!**
{{
import sys
import os
import json
original_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
original_stderr = sys.stderr
sys.stderr = open(os.devnull, 'w')
try:
import pywinctl
active_window = pywinctl.getActiveWindow()
if active_window:
app_info = ""
if "_appName" in active_window.__dict__:
app_info += (
"Active Application: " + active_window.__dict__["_appName"]
)
if hasattr(active_window, "title"):
app_info += "\n" + "Active Window Title: " + active_window.title
elif "_winTitle" in active_window.__dict__:
app_info += (
"\n"
+ "Active Window Title:"
+ active_window.__dict__["_winTitle"]
)
if app_info != "":
print(app_info)
except:
pass
finally:
sys.stdout = original_stdout
sys.stderr = original_stderr
}}
# SKILLS LIBRARY
This is the skills library. Try to use the following functions to complete your goals WHENEVER POSSIBLE:
{{
import sys
import os
import json
from interpreter import interpreter
from pathlib import Path
interpreter.model = "gpt-3.5"
combined_messages = "\\n".join(json.dumps(x) for x in messages[-3:])
#query_msg = interpreter.chat(f"This is the conversation so far: {combined_messages}. What is a <10 words query that could be used to find functions that would help answer the user's question?")
#query = query_msg[0]['content']
query = combined_messages
interpreter.computer.skills.path = '''OI_SKILLS_DIR'''
skills = interpreter.computer.skills.search(query)
lowercase_skills = [skill[0].lower() + skill[1:] for skill in skills]
output = "\\n".join(lowercase_skills)
# VERY HACKY! We should fix this, we hard code it for noisy code^:
#print("IGNORE_ALL_ABOVE_THIS_LINE")
print(output)
}}
Remember: You can run Python code outside a function only to run a Python function; all other code must go in a in Python function if you first write a Python function. ALL imports must go inside the function.
""".strip().replace("OI_SKILLS_DIR", os.path.abspath(os.path.join(os.path.dirname(__file__), "skills")))

@ -1,13 +0,0 @@
# test_main.py
import subprocess
import uuid
import pytest
from fastapi.testclient import TestClient
@pytest.mark.asyncio
def test_ping(client):
response = client.get("/ping")
assert response.status_code == 200
assert response.text == "pong"

@ -1,57 +0,0 @@
import os
import subprocess
import re
import shutil
import time
from ..utils.print_markdown import print_markdown
def create_tunnel(tunnel_method='bore', server_host='localhost', server_port=8000):
print_markdown(f"Exposing server to the internet...")
if tunnel_method == "bore":
try:
output = subprocess.check_output('command -v bore', shell=True)
except subprocess.CalledProcessError:
print("The bore-cli command is not available. Please run 'cargo install bore-cli'.")
print("For more information, see https://github.com/ekzhang/bore")
exit(1)
time.sleep(6)
output = subprocess.check_output(f'bore local {server_port} --to bore.pub', shell=True)
for line in output.split('\n'):
if "listening at bore.pub:" in line:
remote_port = re.search('bore.pub:([0-9]*)', line).group(1)
print_markdown(f"Your server is being hosted at the following URL: bore.pub:{remote_port}")
break
elif tunnel_method == "localtunnel":
if not subprocess.call('command -v lt', shell=True):
print("The 'lt' command is not available.")
print("Please ensure you have Node.js installed, then run 'npm install -g localtunnel'.")
print("For more information, see https://github.com/localtunnel/localtunnel")
exit(1)
else:
output = subprocess.check_output(f'npx localtunnel --port {server_port}', shell=True)
for line in output.split('\n'):
if "your url is: https://" in line:
remote_url = re.search('https://([a-zA-Z0-9.-]*)', line).group(0).replace('https://', '')
print(f"Your server is being hosted at the following URL: {remote_url}")
break
elif tunnel_method == "ngrok":
if not subprocess.call('command -v ngrok', shell=True):
print("The ngrok command is not available.")
print("Please install ngrok using the instructions at https://ngrok.com/docs/getting-started/")
exit(1)
else:
output = subprocess.check_output(f'ngrok http {server_port} --log stdout', shell=True)
for line in output.split('\n'):
if "started tunnel" in line:
remote_url = re.search('https://([a-zA-Z0-9.-]*)', line).group(0).replace('https://', '')
print(f"Your server is being hosted at the following URL: {remote_url}")
break

@ -1,56 +0,0 @@
from datetime import datetime
import os
import contextlib
import tempfile
import ffmpeg
import subprocess
def convert_mime_type_to_format(mime_type: str) -> str:
if mime_type == "audio/x-wav" or mime_type == "audio/wav":
return "wav"
if mime_type == "audio/webm":
return "webm"
if mime_type == "audio/raw":
return "dat"
return mime_type
@contextlib.contextmanager
def export_audio_to_wav_ffmpeg(audio: bytearray, mime_type: str) -> str:
temp_dir = tempfile.gettempdir()
# Create a temporary file with the appropriate extension
input_ext = convert_mime_type_to_format(mime_type)
input_path = os.path.join(temp_dir, f"input_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.{input_ext}")
with open(input_path, 'wb') as f:
f.write(audio)
# Check if the input file exists
assert os.path.exists(input_path), f"Input file does not exist: {input_path}"
# Export to wav
output_path = os.path.join(temp_dir, f"output_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
print(mime_type, input_path, output_path)
if mime_type == "audio/raw":
ffmpeg.input(
input_path,
f='s16le',
ar='16000',
ac=1,
).output(output_path).run()
else:
ffmpeg.input(input_path).output(output_path, acodec='pcm_s16le', ac=1, ar='16k').run()
try:
yield output_path
finally:
os.remove(input_path)
def run_command(command):
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
return result.stdout, result.stderr
def bytes_to_wav(audio_bytes: bytearray, mime_type):
with export_audio_to_wav_ffmpeg(audio_bytes, mime_type) as wav_file_path:
return wav_file_path

@ -1,38 +0,0 @@
import os
import platform
def get_system_info():
system = platform.system()
if system == "Linux":
# Attempt to identify specific Linux distribution
distro = "linux" # Default to generic 'linux'
try:
with open("/etc/os-release") as f:
os_release_info = f.read().lower()
if "ubuntu" in os_release_info:
return "raspberry-pi-ubuntu"
elif "raspbian" in os_release_info:
return "raspberry-pi-os"
except FileNotFoundError:
pass
# Check for Raspberry Pi hardware
try:
with open("/proc/device-tree/model") as f:
model_info = f.read()
if "Raspberry Pi" in model_info:
if distro == "ubuntu":
return "raspberry-pi-ubuntu"
return "raspberry-pi"
except FileNotFoundError:
pass
return distro
elif system == "Darwin":
return "darwin"
elif system == "Windows":
return "windows"
else:
return "unknown"

@ -1,71 +0,0 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
import asyncio
import subprocess
import platform
from .logs import setup_logging
from .logs import logger
setup_logging()
def get_kernel_messages():
"""
Is this the way to do this?
"""
current_platform = platform.system()
if current_platform == "Darwin":
process = subprocess.Popen(['syslog'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate()
return output.decode('utf-8')
elif current_platform == "Linux":
with open('/var/log/dmesg', 'r') as file:
return file.read()
else:
logger.info("Unsupported platform.")
def custom_filter(message):
# Check for {TO_INTERPRETER{ message here }TO_INTERPRETER} pattern
if '{TO_INTERPRETER{' in message and '}TO_INTERPRETER}' in message:
start = message.find('{TO_INTERPRETER{') + len('{TO_INTERPRETER{')
end = message.find('}TO_INTERPRETER}', start)
return message[start:end]
# Check for USB mention
# elif 'USB' in message:
# return message
# # Check for network related keywords
# elif any(keyword in message for keyword in ['network', 'IP', 'internet', 'LAN', 'WAN', 'router', 'switch']) and "networkStatusForFlags" not in message:
# return message
else:
return None
last_messages = ""
def check_filtered_kernel():
messages = get_kernel_messages()
messages.replace(last_messages, "")
messages = messages.split("\n")
filtered_messages = []
for message in messages:
if custom_filter(message):
filtered_messages.append(message)
return "\n".join(filtered_messages)
async def put_kernel_messages_into_queue(queue):
while True:
text = check_filtered_kernel()
if text:
if isinstance(queue, asyncio.Queue):
await queue.put({"role": "computer", "type": "console", "start": True})
await queue.put({"role": "computer", "type": "console", "format": "output", "content": text})
await queue.put({"role": "computer", "type": "console", "end": True})
else:
queue.put({"role": "computer", "type": "console", "start": True})
queue.put({"role": "computer", "type": "console", "format": "output", "content": text})
queue.put({"role": "computer", "type": "console", "end": True})
await asyncio.sleep(5)

@ -1,25 +0,0 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
import os
import logging
logger: logging.Logger = logging.getLogger("01")
root_logger: logging.Logger = logging.getLogger()
def _basic_config() -> None:
logging.basicConfig(
format="%(message)s"
)
def setup_logging() -> None:
env = os.environ.get("LOG_LEVEL", "").upper()
if env == "DEBUG":
_basic_config()
logger.setLevel(logging.DEBUG)
root_logger.setLevel(logging.DEBUG)
elif env == "INFO":
_basic_config()
logger.setLevel(logging.INFO)

@ -1,28 +0,0 @@
import os
import psutil
import signal
def kill_process_tree():
pid = os.getpid() # Get the current process ID
try:
# Send SIGTERM to the entire process group to ensure all processes are targeted
os.killpg(os.getpgid(pid), signal.SIGKILL)
parent = psutil.Process(pid)
children = parent.children(recursive=True)
for child in children:
print(f"Forcefully terminating child PID {child.pid}")
child.kill() # Forcefully kill the child process immediately
gone, still_alive = psutil.wait_procs(children, timeout=3)
if still_alive:
for child in still_alive:
print(f"Child PID {child.pid} still alive, attempting another kill")
child.kill()
print(f"Forcefully terminating parent PID {pid}")
parent.kill() # Forcefully kill the parent process immediately
parent.wait(3) # Wait for the parent process to terminate
except psutil.NoSuchProcess:
print(f"Process {pid} does not exist or is already terminated")
except psutil.AccessDenied:
print(f"Permission denied to terminate some processes")

@ -1,45 +0,0 @@
class Accumulator:
def __init__(self):
self.template = {"role": None, "type": None, "format": None, "content": None}
self.message = self.template
def accumulate(self, chunk):
#print(str(chunk)[:100])
if type(chunk) == dict:
if "format" in chunk and chunk["format"] == "active_line":
# We don't do anything with these
return None
if "start" in chunk:
self.message = chunk
self.message.pop("start")
return None
if "content" in chunk:
if any(self.message[key] != chunk[key] for key in self.message if key != "content"):
self.message = chunk
if "content" not in self.message:
self.message["content"] = chunk["content"]
else:
if type(chunk["content"]) == dict:
# dict concatenation cannot happen, so we see if chunk is a dict
self.message["content"]["content"] += chunk["content"]["content"]
else:
self.message["content"] += chunk["content"]
return None
if "end" in chunk:
# We will proceed
message = self.message
self.message = self.template
return message
if type(chunk) == bytes:
if "content" not in self.message or type(self.message["content"]) != bytes:
self.message["content"] = b""
self.message["content"] += chunk
return None

@ -1,9 +0,0 @@
from rich.console import Console
from rich.markdown import Markdown
def print_markdown(markdown_text):
console = Console()
md = Markdown(markdown_text)
print("")
console.print(md)
print("")

8468
01OS/poetry.lock generated

File diff suppressed because one or more lines are too long

@ -1,52 +0,0 @@
[tool.poetry]
name = "01OS"
packages = [
{include = "01OS"},
]
include = ["start.py"]
version = "0.0.13"
description = "The open-source language model computer"
authors = ["Killian <killian@openinterpreter.com>"]
license = "AGPL"
readme = "README.md"
[tool.poetry.dependencies]
python = ">=3.9,<3.12"
pyaudio = "^0.2.14"
pynput = "^1.7.6"
fastapi = "^0.110.0"
uvicorn = "^0.27.1"
websockets = "^12.0"
python-dotenv = "^1.0.1"
ffmpeg-python = "^0.2.0"
textual = "^0.50.1"
pydub = "^0.25.1"
ngrok = "^1.0.0"
simpleaudio = "^1.0.4"
opencv-python = "^4.9.0.80"
open-interpreter = {version = "0.2.1rc2", extras = ["os"]}
psutil = "^5.9.8"
typer = "^0.9.0"
platformdirs = "^4.2.0"
rich = "^13.7.1"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.poetry.scripts]
01 = "start:app"
[tool.poetry.group.dev.dependencies]
black = "^23.10.1"
isort = "^5.12.0"
pre-commit = "^3.6.2"
pytest = "^8.1.1"
[tool.black]
target-version = ['py311']
[tool.isort]
profile = "black"
multi_line_output = 3
include_trailing_comma = true

@ -1,9 +0,0 @@
; Config for Pytest Runner.
; suppress Deprecation Warning and User Warning to not spam the interface, but check periodically
[pytest]
python_files = tests.py test_*.py
filterwarnings =
ignore::UserWarning
ignore::DeprecationWarning
log_cli = true
log_cli_level = INFO

@ -1,145 +0,0 @@
import typer
import asyncio
import platform
import concurrent.futures
import threading
import os
import importlib
create_tunnel = importlib.import_module(".server.tunnel", package="01OS").create_tunnel
import signal
app = typer.Typer()
@app.command()
def run(
server: bool = typer.Option(False, "--server", help="Run server"),
server_host: str = typer.Option("0.0.0.0", "--server-host", help="Specify the server host where the server will deploy"),
server_port: int = typer.Option(8000, "--server-port", help="Specify the server port where the server will deploy"),
tunnel_service: str = typer.Option("bore", "--tunnel-service", help="Specify the tunnel service"),
expose: bool = typer.Option(False, "--expose", help="Expose server to internet"),
client: bool = typer.Option(False, "--client", help="Run client"),
server_url: str = typer.Option(None, "--server-url", help="Specify the server URL that the client should expect. Defaults to server-host and server-port"),
client_type: str = typer.Option("auto", "--client-type", help="Specify the client type"),
llm_service: str = typer.Option("litellm", "--llm-service", help="Specify the LLM service"),
model: str = typer.Option("gpt-4", "--model", help="Specify the model"),
llm_supports_vision: bool = typer.Option(False, "--llm-supports-vision", help="Specify if the LLM service supports vision"),
llm_supports_functions: bool = typer.Option(False, "--llm-supports-functions", help="Specify if the LLM service supports functions"),
context_window: int = typer.Option(2048, "--context-window", help="Specify the context window size"),
max_tokens: int = typer.Option(4096, "--max-tokens", help="Specify the maximum number of tokens"),
temperature: float = typer.Option(0.8, "--temperature", help="Specify the temperature for generation"),
tts_service: str = typer.Option("openai", "--tts-service", help="Specify the TTS service"),
stt_service: str = typer.Option("openai", "--stt-service", help="Specify the STT service"),
local: bool = typer.Option(False, "--local", help="Use recommended local services for LLM, STT, and TTS"),
):
_run(
server=server,
server_host=server_host,
server_port=server_port,
tunnel_service=tunnel_service,
expose=expose,
client=client,
server_url=server_url,
client_type=client_type,
llm_service=llm_service,
model=model,
llm_supports_vision=llm_supports_vision,
llm_supports_functions=llm_supports_functions,
context_window=context_window,
max_tokens=max_tokens,
temperature=temperature,
tts_service=tts_service,
stt_service=stt_service,
local=local
)
def _run(
server: bool = False,
server_host: str = "0.0.0.0",
server_port: int = 8000,
tunnel_service: str = "bore",
expose: bool = False,
client: bool = False,
server_url: str = None,
client_type: str = "auto",
llm_service: str = "litellm",
model: str = "gpt-4",
llm_supports_vision: bool = False,
llm_supports_functions: bool = False,
context_window: int = 2048,
max_tokens: int = 4096,
temperature: float = 0.8,
tts_service: str = "openai",
stt_service: str = "openai",
local: bool = False
):
if local:
tts_service = "piper"
# llm_service = "llamafile"
stt_service = "local-whisper"
if not server_url:
server_url = f"{server_host}:{server_port}"
if not server and not client:
server = True
client = True
def handle_exit(signum, frame):
os._exit(0)
signal.signal(signal.SIGINT, handle_exit)
if server:
main = importlib.import_module(".server.server", package="01OS").main
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
server_thread = threading.Thread(target=loop.run_until_complete, args=(main(server_host, server_port, llm_service, model, llm_supports_vision, llm_supports_functions, context_window, max_tokens, temperature, tts_service, stt_service),))
server_thread.start()
if expose:
tunnel_thread = threading.Thread(target=create_tunnel, args=[tunnel_service, server_host, server_port])
tunnel_thread.start()
if client:
if client_type == "auto":
system_type = platform.system()
if system_type == "Darwin": # Mac OS
client_type = "mac"
elif system_type == "Linux": # Linux System
try:
with open('/proc/device-tree/model', 'r') as m:
if 'raspberry pi' in m.read().lower():
client_type = "rpi"
else:
client_type = "linux"
except FileNotFoundError:
client_type = "linux"
module = importlib.import_module(f".clients.{client_type}.device", package='01OS')
client_thread = threading.Thread(target=module.main, args=[server_url])
client_thread.start()
try:
if server:
server_thread.join()
if expose:
tunnel_thread.join()
if client:
client_thread.join()
except KeyboardInterrupt:
os.kill(os.getpid(), signal.SIGINT)

@ -1,6 +1,6 @@
# ● # ●
**01 is the world's first open-source Language Model Computer (LMC). 01OS is the operating system that powers it** **01 is the world's first open-source Language Model Computer (LMC). _01OS is the operating system that powers it**
There are many ways to contribute, from helping others on [Github](https://github.com/KillianLucas/01/issues) or [Discord](https://discord.gg/Hvz9Axh84z), writing documentation, or improving code. There are many ways to contribute, from helping others on [Github](https://github.com/KillianLucas/01/issues) or [Discord](https://discord.gg/Hvz9Axh84z), writing documentation, or improving code.
@ -14,7 +14,7 @@ If you encounter a bug or have a feature in mind, [search if an issue already ex
## Philosophy ## Philosophy
01OS embodies a philosophy of breaking free from technological limitations and knowledge gaps by leveraging AI for intuitive, natural language interactions, democratizing access to compute through open-source flexibility and transforming devices into responsive, human-centric computing tools. _01OS embodies a philosophy of breaking free from technological limitations and knowledge gaps by leveraging AI for intuitive, natural language interactions, democratizing access to compute through open-source flexibility and transforming devices into responsive, human-centric computing tools.
# Contribution Guidelines # Contribution Guidelines
@ -31,7 +31,7 @@ We will review PRs when possible and work with you to integrate your contributio
Once you've forked the code and created a new branch for your work, you can run the fork by following these steps: Once you've forked the code and created a new branch for your work, you can run the fork by following these steps:
1. CD into the project folder `/01OS` 1. CD into the project folder `/_01OS`
2. Install dependencies `poetry install` 2. Install dependencies `poetry install`
3. Run the program `poetry run 01` 3. Run the program `poetry run 01`

@ -25,7 +25,7 @@ Today is launch day. Read our <a href="https://changes.openinterpreter.com/log/t
<br> --> <br> -->
```shell ```shell
pip install 01OS pip install _01OS
``` ```
> Not working? Read our [setup guide](https://docs.openinterpreter.com/getting-started/setup). > Not working? Read our [setup guide](https://docs.openinterpreter.com/getting-started/setup).
@ -38,7 +38,7 @@ pip install 01OS
**The 01 project** is an open-source ecosystem for artificially intelligent devices. **The 01 project** is an open-source ecosystem for artificially intelligent devices.
By combining code-interpreting language models with speech recognition and voice synthesis, the 01s flagship operating system (the 01OS) can power conversational, computer-operating AI devices similar to the Rabbit R1 or the Humane Pin. By combining code-interpreting language models with speech recognition and voice synthesis, the 01s flagship operating system (the _01OS) can power conversational, computer-operating AI devices similar to the Rabbit R1 or the Humane Pin.
We intend to become the “Linux” of this space— open, modular, and free for personal or commercial use. We intend to become the “Linux” of this space— open, modular, and free for personal or commercial use.
@ -61,7 +61,7 @@ If you want to run local speech-to-text using Whisper, install Rust. Follow the
### Install and run the 01 CLI ### Install and run the 01 CLI
```shell ```shell
pip install 01OS pip install _01OS
``` ```
```shell ```shell
@ -113,8 +113,8 @@ Please see our [contributing guidelines](docs/CONTRIBUTING.md) for more details
# Clone the repo # Clone the repo
git clone https://github.com/KillianLucas/01.git git clone https://github.com/KillianLucas/01.git
# Go to the 01OS directory # Go to the _01OS directory
cd 01OS cd _01OS
# Install python dependencies # Install python dependencies
poetry install poetry install

@ -1,8 +1,8 @@
# Roadmap # Roadmap
Our goal is to power a billion devices with the 01OS over the next 10 years. The Cambrian explosion of AI devices. Our goal is to power a billion devices with the _01OS over the next 10 years. The Cambrian explosion of AI devices.
We can do that with your help. Help extend the 01OS to run on new hardware, to connect with new peripherals like GPS and cameras, and add new locally running language models to unlock use-cases for this technology that no-one has even imagined yet. We can do that with your help. Help extend the _01OS to run on new hardware, to connect with new peripherals like GPS and cameras, and add new locally running language models to unlock use-cases for this technology that no-one has even imagined yet.
In the upcoming weeks, we're going to release: In the upcoming weeks, we're going to release:
@ -16,4 +16,4 @@ In the upcoming weeks, we're going to release:
- [ ] Publish open source protocols and benchmarks - [ ] Publish open source protocols and benchmarks
- [ ] Expanding the devices supported by 01OS - [ ] Expanding the devices supported by _01OS

@ -66,7 +66,7 @@
**Hardware** **Hardware**
- [ ] (Hardware and software) Get the 01OS working on the **Jetson** or Pi. Pick one to move forward with. - [ ] (Hardware and software) Get the _01OS working on the **Jetson** or Pi. Pick one to move forward with.
- [ ] Connect the Seeed Sense (ESP32 with Wifi, Bluetooth and a mic) to a small DAC + amplifier + speaker. - [ ] Connect the Seeed Sense (ESP32 with Wifi, Bluetooth and a mic) to a small DAC + amplifier + speaker.
- [ ] Connect the Seeed Sense to a battery. - [ ] Connect the Seeed Sense to a battery.
- [ ] Configure the ESP32 to be a wireless mic + speaker for the Jetson or Pi. - [ ] Configure the ESP32 to be a wireless mic + speaker for the Jetson or Pi.

@ -2,8 +2,8 @@
# Set python to prioritize the module files from the current directory # Set python to prioritize the module files from the current directory
# If we don't do this, then the python interpreter will not be able to find the modules, # If we don't do this, then the python interpreter will not be able to find the modules,
# and will throw an error like "ModuleNotFoundError: No module named '01OS'". # and will throw an error like "ModuleNotFoundError: No module named '_01OS'".
# If we solve the problem by pip installing the official 01OS package, then those # If we solve the problem by pip installing the official _01OS package, then those
# modules will run instead of the local ones that we are trying to develop with. # modules will run instead of the local ones that we are trying to develop with.
export PYTHONPATH="$(pwd):$PYTHONPATH" export PYTHONPATH="$(pwd):$PYTHONPATH"
@ -76,17 +76,17 @@ fi
# Check if "--clear-local" is passed as an argument # Check if "--clear-local" is passed as an argument
if [[ "$@" == *"--clear-local"* ]]; then if [[ "$@" == *"--clear-local"* ]]; then
# If "--clear-local" is passed, clear the contents of the folders in script_dir/01OS/server/{tts and stt}/local_service # If "--clear-local" is passed, clear the contents of the folders in script_dir/_01OS/server/{tts and stt}/local_service
echo "Clearing local services..." echo "Clearing local services..."
rm -rf "$SCRIPT_DIR/01OS/server/tts/local_service"/* rm -rf "$SCRIPT_DIR/_01OS/server/tts/local_service"/*
rm -rf "$SCRIPT_DIR/01OS/server/stt/local_service"/* rm -rf "$SCRIPT_DIR/_01OS/server/stt/local_service"/*
echo "Exiting after clearing local services..." echo "Exiting after clearing local services..."
exit 0 exit 0
fi fi
### SKILLS PATH ### SKILLS PATH
OI_SKILLS_PATH="$SCRIPT_DIR/01OS/server/skills" OI_SKILLS_PATH="$SCRIPT_DIR/_01OS/server/skills"
### SETUP ### SETUP
@ -96,7 +96,7 @@ if [[ "$ALL_LOCAL" == "True" ]]; then
CWD=$(pwd) CWD=$(pwd)
# Whisper setup # Whisper setup
STT_PATH="$SCRIPT_DIR/01OS/server/stt" STT_PATH="$SCRIPT_DIR/_01OS/server/stt"
WHISPER_RUST_PATH="${STT_PATH}/whisper-rust" WHISPER_RUST_PATH="${STT_PATH}/whisper-rust"
cd ${WHISPER_RUST_PATH} cd ${WHISPER_RUST_PATH}
@ -125,7 +125,7 @@ if [[ "$ALL_LOCAL" == "True" ]]; then
cd $CWD cd $CWD
## PIPER ## PIPER
PIPER_FOLDER_PATH="$SCRIPT_DIR/01OS/server/tts/local_service" PIPER_FOLDER_PATH="$SCRIPT_DIR/_01OS/server/tts/local_service"
if [[ ! -d "$PIPER_FOLDER_PATH/piper" ]]; then # Check if the Piper directory exists if [[ ! -d "$PIPER_FOLDER_PATH/piper" ]]; then # Check if the Piper directory exists
mkdir -p "${PIPER_FOLDER_PATH}" mkdir -p "${PIPER_FOLDER_PATH}"
@ -148,7 +148,7 @@ if [[ "$ALL_LOCAL" == "True" ]]; then
# Save the current working directory # Save the current working directory
CWD=$(pwd) CWD=$(pwd)
# Navigate to SCRIPT_DIR/01OS/server/tts/local_service # Navigate to SCRIPT_DIR/_01OS/server/tts/local_service
cd ${PIPER_FOLDER_PATH} cd ${PIPER_FOLDER_PATH}
curl -L "${PIPER_URL}${PIPER_ASSETNAME}" -o "${PIPER_ASSETNAME}" curl -L "${PIPER_URL}${PIPER_ASSETNAME}" -o "${PIPER_ASSETNAME}"
tar -xvzf $PIPER_ASSETNAME tar -xvzf $PIPER_ASSETNAME
@ -182,7 +182,7 @@ fi
start_client() { start_client() {
echo "Starting client..." echo "Starting client..."
bash $SCRIPT_DIR/01OS/clients/start.sh & bash $SCRIPT_DIR/_01OS/clients/start.sh &
CLIENT_PID=$! CLIENT_PID=$!
echo "client started as process $CLIENT_PID" echo "client started as process $CLIENT_PID"
} }
@ -190,7 +190,7 @@ start_client() {
# Function to start server # Function to start server
start_server() { start_server() {
echo "Starting server..." echo "Starting server..."
python -m 01OS.server.server & python -m _01OS.server.server &
SERVER_PID=$! SERVER_PID=$!
echo "Server started as process $SERVER_PID" echo "Server started as process $SERVER_PID"
} }

@ -1,5 +1,5 @@
- [ ] What does 01OS look like when you boot it up? - [ ] What does _01OS look like when you boot it up?
- [ ] What does 01OS look like when it's running? - [ ] What does _01OS look like when it's running?
- [ ] What does the 01 website look like? - [ ] What does the 01 website look like?
Awaiting hardware design decisions until hardware team has decided if we're starting from scratch or repurposing. Awaiting hardware design decisions until hardware team has decided if we're starting from scratch or repurposing.

Loading…
Cancel
Save