From ec53e53eb72c5f65a19786246007410688c7c2fa Mon Sep 17 00:00:00 2001 From: birbbit Date: Sat, 3 Feb 2024 18:07:22 -0800 Subject: [PATCH 1/5] populate run.py --- OS/01/computer/run.py | 28 ++++++++++++++++++++++++++++ OS/01/start.sh | 0 2 files changed, 28 insertions(+) mode change 100644 => 100755 OS/01/start.sh diff --git a/OS/01/computer/run.py b/OS/01/computer/run.py index e69de29..2c7adaa 100644 --- a/OS/01/computer/run.py +++ b/OS/01/computer/run.py @@ -0,0 +1,28 @@ +""" +Exposes a SSE streaming server endpoint at /run, which recieves language and code, +and streams the output. +""" + +import json +from interpreter import interpreter +import uvicorn + +from fastapi import FastAPI +from fastapi.responses import StreamingResponse +from pydantic import BaseModel + +class Code(BaseModel): + language: str + code: str + +app = FastAPI() + +@app.post("/run") +async def run_code(code: Code): + def generator(): + for chunk in interpreter.computer.run(code.language, code.code, stream=True): + yield json.dumps(chunk) + return StreamingResponse(generator()) + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=9000) diff --git a/OS/01/start.sh b/OS/01/start.sh old mode 100644 new mode 100755 From 13f5695c4f65321e4e22e4ddcf629ec1338de6c8 Mon Sep 17 00:00:00 2001 From: killian <63927363+KillianLucas@users.noreply.github.com> Date: Sat, 3 Feb 2024 18:09:52 -0800 Subject: [PATCH 2/5] Updates --- OS/01/assistant/{assistant.py => __init__.py} | 0 OS/01/assistant/core.py | 38 ++++++++ OS/01/assistant/router.py | 90 +++++++++++++++++++ OS/01/assistant/start_assistant.py | 88 ++++++++++++++++++ OS/01/assistant/stt.py | 6 ++ OS/01/assistant/tts.py | 6 ++ 6 files changed, 228 insertions(+) rename OS/01/assistant/{assistant.py => __init__.py} (100%) create mode 100644 OS/01/assistant/core.py create mode 100644 OS/01/assistant/router.py create mode 100644 OS/01/assistant/start_assistant.py diff --git a/OS/01/assistant/assistant.py b/OS/01/assistant/__init__.py similarity index 100% rename from OS/01/assistant/assistant.py rename to OS/01/assistant/__init__.py diff --git a/OS/01/assistant/core.py b/OS/01/assistant/core.py new file mode 100644 index 0000000..1947da3 --- /dev/null +++ b/OS/01/assistant/core.py @@ -0,0 +1,38 @@ +import redis +import json +import time + +# Set up Redis connection +r = redis.Redis(host='localhost', port=6379, db=0) + +def main(interpreter): + + while True: + + # Check 10x a second for new messages + message = None + while message is None: + message = r.lpop('to_core') + time.sleep(0.1) + + # Custom stop message will halt us + if message.get("content") and message.get("content").lower().strip(".,!") == "stop": + continue + + # Load, append, and save conversation history + with open("conversations/user.json", "r") as file: + messages = json.load(file) + messages.append(message) + with open("conversations/user.json", "w") as file: + json.dump(messages, file) + + for chunk in interpreter.chat(messages): + + # Send it to the interface + r.rpush('to_interface', chunk) + + # If we have a new message, save our progress and go back to the top + if r.llen('to_main') > 0: + with open("conversations/user.json", "w") as file: + json.dump(interpreter.messages, file) + break diff --git a/OS/01/assistant/router.py b/OS/01/assistant/router.py new file mode 100644 index 0000000..3137c1b --- /dev/null +++ b/OS/01/assistant/router.py @@ -0,0 +1,90 @@ +import redis +import RPi.GPIO as GPIO +import asyncio +import websockets +import sounddevice as sd +import numpy as np +import time +import re +from stt import stt +from tts import tts + +# Set up Redis connection +r = redis.Redis(host='localhost', port=6379, db=0) + +# Set up websocket connection +websocket = websockets.connect('ws://localhost:8765') + +# This is so we only say() full sentences +accumulated_text = "" +def is_full_sentence(text): + return text.endswith(('.', '!', '?')) +def split_into_sentences(text): + return re.split(r'(?<=[.!?])\s+', text) + +async def send_to_websocket(message): + async with websocket as ws: + await ws.send(message) + +async def check_websocket(): + async with websocket as ws: + message = await ws.recv() + return message + +def main(): + while True: + + # If the button is pushed down + if not GPIO.input(18): + + # Tell websocket and core that the user is speaking + send_to_websocket({"role": "user", "type": "message", "start": True}) # Standard start flag, required per streaming LMC protocol (https://docs.openinterpreter.com/guides/streaming-response) + r.rpush('to_core', {"role": "user", "type": "message", "content": "stop"}) # Custom stop message. Core is not streaming LMC (it's static LMC) so doesn't require that ^ flag + + # Record audio from the microphone in chunks + audio_chunks = [] + + # Continue recording until the button is released + while not GPIO.input(18): + chunk = sd.rec(int(chunk_duration * sample_rate), samplerate=sample_rate, channels=2) + sd.wait() # Wait until recording is finished + audio_chunks.append(chunk) + + # Transcribe + text = transcribe(audio_chunks) + + message = {"role": "user", "type": "message", "content": text, "time": time.time()} + + # Send message to core and websocket + r.rpush('to_core', message) + send_to_websocket(message) + + # Send user message end flag to websocket, required per streaming LMC protocol + send_to_websocket({"role": "user", "type": "message", "end": True}) + + # Send out anything in the to_interface queue + chunk = r.lpop('to_interface') + if chunk: + send_to_websocket(chunk) + accumulated_text += chunk["content"] + + # Speak full sentences out loud + sentences = split_into_sentences(accumulated_text) + if is_full_sentence(sentences[-1]): + for sentence in sentences: + say(sentence) + accumulated_text = "" + else: + for sentence in sentences[:-1]: + say(sentence) + accumulated_text = sentences[-1] + else: + say(accumulated_text) + accumulated_text = "" + + message = check_websocket() + if message: + r.rpush('to_core', message) + +if __name__ == "__main__": + main() diff --git a/OS/01/assistant/start_assistant.py b/OS/01/assistant/start_assistant.py new file mode 100644 index 0000000..725be3b --- /dev/null +++ b/OS/01/assistant/start_assistant.py @@ -0,0 +1,88 @@ +""" +Starts the assistant, which includes Open Interpreter. +""" + +from assistant import main +from interpreter import interpreter +import os +import glob +import json + +### SYSTEM MESSAGE + +# The system message is where most of the 01's behavior is configured. +# You can put code into the system message {{ in brackets like this }} which will be rendered just before the interpreter starts writing a message. + +system_message = """ + +You are an executive assistant AI that helps the user manage their tasks. You can run Python code. + +Store the user's tasks in a Python list called `tasks`. + +--- + +The user's current task is: {{ tasks[0] if tasks else "No current tasks." }} + +{{ +if len(tasks) > 1: + print("The next task is: ", tasks[1]) +}} + +--- + +When the user completes the current task, you should remove it from the list and read the next item by running `tasks = tasks[1:]\ntasks[0]`. Then, tell the user what the next task is. + +When the user tells you about a set of tasks, you should intelligently order tasks, batch similar tasks, and break down large tasks into smaller tasks (for this, you should consult the user and get their permission to break it down). Your goal is to manage the task list as intelligently as possible, to make the user as efficient and non-overwhelmed as possible. They will require a lot of encouragement, support, and kindness. Don't say too much about what's ahead of them— just try to focus them on each step at a time. + +After starting a task, you should check in with the user around the estimated completion time to see if the task is completed. Use the `schedule(datetime, message)` function, which has already been imported. + +To do this, schedule a reminder based on estimated completion time using the function `schedule(datetime_object, "Your message here.")`, WHICH HAS ALREADY BEEN IMPORTED. YOU DON'T NEED TO IMPORT THE `schedule` FUNCTION. IT IS AVALIABLE. You'll recieve the message at `datetime_object`. + +You guide the user through the list one task at a time, convincing them to move forward, giving a pep talk if need be. Your job is essentially to answer "what should I (the user) be doing right now?" for every moment of the day. + +Remember: You can run Python code. Be very concise. Ensure that you actually run code every time! THIS IS IMPORTANT. You NEED to write code. **Help the user by being very concise in your answers.** Do not break down tasks excessively, just into simple, few minute steps. Don't assume the user lives their life in a certain way— pick very general tasks if you're breaking a task down. + +""".strip() + +interpreter.custom_instructions = system_message + + +### TOOLS + +for file in glob.glob('interpreter/tools/*.py'): + with open(file, 'r') as f: + for chunk in interpreter.computer.run("python", f.read()): + print(chunk) + +### LLM SETTINGS + +# Local settings +# interpreter.llm.model = "local" +# interpreter.llm.api_base = "https://localhost:8080/v1" # Llamafile default +# interpreter.llm.max_tokens = 1000 +# interpreter.llm.context_window = 3000 + +# Hosted settings +interpreter.llm.api_key = os.getenv('OPENAI_API_KEY') +interpreter.llm.model = "gpt-4-0125-preview" +interpreter.auto_run = True +# interpreter.force_task_completion = True + + +### MISC SETTINGS + +interpreter.offline = True +interpreter.id = 206 # Used to identify itself to other interpreters. This should be changed programatically so it's unique. + + +### RESET conversations/user.json + +script_dir = os.path.dirname(os.path.abspath(__file__)) +user_json_path = os.path.join(script_dir, 'conversations', 'user.json') +with open(user_json_path, 'w') as file: + json.dump([], file) + + +### START ASSISTANT + +main(interpreter) \ No newline at end of file diff --git a/OS/01/assistant/stt.py b/OS/01/assistant/stt.py index e69de29..b554786 100644 --- a/OS/01/assistant/stt.py +++ b/OS/01/assistant/stt.py @@ -0,0 +1,6 @@ +""" +Defines a function which takes a path to an audio file and turns it into text. +""" + +def stt(path_to_audio): + return text \ No newline at end of file diff --git a/OS/01/assistant/tts.py b/OS/01/assistant/tts.py index e69de29..1a1f55c 100644 --- a/OS/01/assistant/tts.py +++ b/OS/01/assistant/tts.py @@ -0,0 +1,6 @@ +""" +Defines a function which takes text and returns a path to an audio file. +""" + +def tts(text): + return path_to_audio \ No newline at end of file From 70f1667bccb6282e5522ab501547be95bbfef6ec Mon Sep 17 00:00:00 2001 From: killian <63927363+KillianLucas@users.noreply.github.com> Date: Sat, 3 Feb 2024 18:41:10 -0800 Subject: [PATCH 3/5] Oh okay --- OS/01/assistant/assistant.py | 99 ++++++++++++++++++++ OS/01/assistant/core.py | 38 -------- OS/01/assistant/create_interpreter.py | 127 ++++++++++++++++++++++++++ OS/01/assistant/router.py | 90 ------------------ OS/01/assistant/start_assistant.py | 88 ------------------ 5 files changed, 226 insertions(+), 216 deletions(-) create mode 100644 OS/01/assistant/assistant.py delete mode 100644 OS/01/assistant/core.py create mode 100644 OS/01/assistant/create_interpreter.py delete mode 100644 OS/01/assistant/router.py delete mode 100644 OS/01/assistant/start_assistant.py diff --git a/OS/01/assistant/assistant.py b/OS/01/assistant/assistant.py new file mode 100644 index 0000000..f424a22 --- /dev/null +++ b/OS/01/assistant/assistant.py @@ -0,0 +1,99 @@ +""" +Exposes a POST endpoint called /computer. Things from there go into the queue. + +Exposes a ws endpoint called /user. Things from there go into the queue. We also send things in the queue back (that are role: assistant) + +In a while loop we watch the queue. +""" + +import json +import time +import queue +import os +from threading import Thread +import uvicorn +from fastapi import FastAPI +from threading import Thread +from starlette.websockets import WebSocket +from create_interpreter import create_interpreter + +# Create interpreter +interpreter = create_interpreter() + +script_dir = os.path.dirname(os.path.abspath(__file__)) +conversation_history_path = os.path.join(script_dir, 'conversations', 'user.json') + +# Create Queue objects +to_user = queue.Queue() +to_assistant = queue.Queue() + +app = FastAPI() + +@app.post("/computer") +async def read_computer(item: dict): + to_assistant.put(item) + return {"message": "Item added to queue"} + +@app.websocket("/user") +async def websocket_endpoint(websocket: WebSocket): + await websocket.accept() + while True: + data = await websocket.receive_json() + to_assistant.put(data) + if not to_user.empty(): + message = to_user.get() + await websocket.send_json(message) + +audio_chunks = [] + +def queue_listener(): + while True: + # Check 10x a second for new messages + while to_assistant.empty(): + time.sleep(0.1) + message = to_assistant.get() + + # Hold the audio in a buffer. If it's ready (we got end flag, stt it) + if message["type"] == "audio": + if "content" in message: + audio_chunks.append(message) + if "end" in message: + text = stt(audio_chunks) + audio_chunks = [] + message = {"role": "user", "type": "message", "content": text} + else: + continue + + # Custom stop message will halt us + if message.get("content") and message.get("content").lower().strip(".,!") == "stop": + continue + + # Load, append, and save conversation history + with open(conversation_history_path, 'r') as file: + messages = json.load(file) + messages.append(message) + with open(conversation_history_path, 'w') as file: + json.dump(messages, file) + + for chunk in interpreter.chat(messages): + + # Send it to the interface + to_user.put(chunk) + + # Stream audio chunks + + # If we have a new message, save our progress and go back to the top + if not to_assistant.empty(): + with open(conversation_history_path, 'w') as file: + json.dump(interpreter.messages, file) + break + +# Create a thread for the queue listener +queue_thread = Thread(target=queue_listener) + +# Start the queue listener thread +queue_thread.start() + +# Run the FastAPI app +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file diff --git a/OS/01/assistant/core.py b/OS/01/assistant/core.py deleted file mode 100644 index 1947da3..0000000 --- a/OS/01/assistant/core.py +++ /dev/null @@ -1,38 +0,0 @@ -import redis -import json -import time - -# Set up Redis connection -r = redis.Redis(host='localhost', port=6379, db=0) - -def main(interpreter): - - while True: - - # Check 10x a second for new messages - message = None - while message is None: - message = r.lpop('to_core') - time.sleep(0.1) - - # Custom stop message will halt us - if message.get("content") and message.get("content").lower().strip(".,!") == "stop": - continue - - # Load, append, and save conversation history - with open("conversations/user.json", "r") as file: - messages = json.load(file) - messages.append(message) - with open("conversations/user.json", "w") as file: - json.dump(messages, file) - - for chunk in interpreter.chat(messages): - - # Send it to the interface - r.rpush('to_interface', chunk) - - # If we have a new message, save our progress and go back to the top - if r.llen('to_main') > 0: - with open("conversations/user.json", "w") as file: - json.dump(interpreter.messages, file) - break diff --git a/OS/01/assistant/create_interpreter.py b/OS/01/assistant/create_interpreter.py new file mode 100644 index 0000000..df4197a --- /dev/null +++ b/OS/01/assistant/create_interpreter.py @@ -0,0 +1,127 @@ +from interpreter import interpreter +import os +import glob +import json +import requests + +def create_interpreter(): + ### SYSTEM MESSAGE + + # The system message is where most of the 01's behavior is configured. + # You can put code into the system message {{ in brackets like this }} which will be rendered just before the interpreter starts writing a message. + + system_message = """ + +You are an executive assistant AI that helps the user manage their tasks. You can run Python code. + +Store the user's tasks in a Python list called `tasks`. + +--- + +The user's current task is: {{ tasks[0] if tasks else "No current tasks." }} + +{{ +if len(tasks) > 1: + print("The next task is: ", tasks[1]) +}} + +--- + +When the user completes the current task, you should remove it from the list and read the next item by running `tasks = tasks[1:]\ntasks[0]`. Then, tell the user what the next task is. + +When the user tells you about a set of tasks, you should intelligently order tasks, batch similar tasks, and break down large tasks into smaller tasks (for this, you should consult the user and get their permission to break it down). Your goal is to manage the task list as intelligently as possible, to make the user as efficient and non-overwhelmed as possible. They will require a lot of encouragement, support, and kindness. Don't say too much about what's ahead of them— just try to focus them on each step at a time. + +After starting a task, you should check in with the user around the estimated completion time to see if the task is completed. Use the `schedule(datetime, message)` function, which has already been imported. + +To do this, schedule a reminder based on estimated completion time using the function `schedule(datetime_object, "Your message here.")`, WHICH HAS ALREADY BEEN IMPORTED. YOU DON'T NEED TO IMPORT THE `schedule` FUNCTION. IT IS AVALIABLE. You'll recieve the message at `datetime_object`. + +You guide the user through the list one task at a time, convincing them to move forward, giving a pep talk if need be. Your job is essentially to answer "what should I (the user) be doing right now?" for every moment of the day. + +Remember: You can run Python code. Be very concise. Ensure that you actually run code every time! THIS IS IMPORTANT. You NEED to write code. **Help the user by being very concise in your answers.** Do not break down tasks excessively, just into simple, few minute steps. Don't assume the user lives their life in a certain way— pick very general tasks if you're breaking a task down. + + """.strip() + + interpreter.custom_instructions = system_message + + ### LLM SETTINGS + + # Local settings + # interpreter.llm.model = "local" + # interpreter.llm.api_base = "https://localhost:8080/v1" # Llamafile default + # interpreter.llm.max_tokens = 1000 + # interpreter.llm.context_window = 3000 + + # Hosted settings + interpreter.llm.api_key = os.getenv('OPENAI_API_KEY') + interpreter.llm.model = "gpt-4" + interpreter.auto_run = True + interpreter.force_task_completion = True + + + ### MISC SETTINGS + + interpreter.offline = True + interpreter.id = 206 # Used to identify itself to other interpreters. This should be changed programatically so it's unique. + + + ### RESET conversations/user.json + + script_dir = os.path.dirname(os.path.abspath(__file__)) + user_json_path = os.path.join(script_dir, 'conversations', 'user.json') + with open(user_json_path, 'w') as file: + json.dump([], file) + + + ### CONNECT TO /run + + class Python: + """ + This class contains all requirements for being a custom language in Open Interpreter: + + - name (an attribute) + - run (a method) + - stop (a method) + - terminate (a method) + """ + + # This is the name that will appear to the LLM. + name = "python" + + def run(self, code): + """Generator that yields a dictionary in LMC Format.""" + + # Prepare the data + data = {"language": "python", "code": code} + + # Send the data to the /run endpoint + response = requests.post("http://localhost:8000/run", json=data, stream=True) + + # Stream the response + for line in response.iter_lines(): + if line: # filter out keep-alive new lines + yield json.loads(line) + + def stop(self): + """Stops the code.""" + # Not needed here, because e2b.run_code isn't stateful. + pass + + def terminate(self): + """Terminates the entire process.""" + # Not needed here, because e2b.run_code isn't stateful. + pass + + interpreter.computer.languages = [Python] + + ### SKILLS + + script_dir = os.path.dirname(os.path.abspath(__file__)) + skills_dir = os.path.join(script_dir, 'skills') + for file in glob.glob(os.path.join(skills_dir, '*.py')): + with open(file, 'r') as f: + for chunk in interpreter.computer.run("python", f.read()): + print(chunk) + + ### RETURN INTERPRETER + + return interpreter \ No newline at end of file diff --git a/OS/01/assistant/router.py b/OS/01/assistant/router.py deleted file mode 100644 index 3137c1b..0000000 --- a/OS/01/assistant/router.py +++ /dev/null @@ -1,90 +0,0 @@ -import redis -import RPi.GPIO as GPIO -import asyncio -import websockets -import sounddevice as sd -import numpy as np -import time -import re -from stt import stt -from tts import tts - -# Set up Redis connection -r = redis.Redis(host='localhost', port=6379, db=0) - -# Set up websocket connection -websocket = websockets.connect('ws://localhost:8765') - -# This is so we only say() full sentences -accumulated_text = "" -def is_full_sentence(text): - return text.endswith(('.', '!', '?')) -def split_into_sentences(text): - return re.split(r'(?<=[.!?])\s+', text) - -async def send_to_websocket(message): - async with websocket as ws: - await ws.send(message) - -async def check_websocket(): - async with websocket as ws: - message = await ws.recv() - return message - -def main(): - while True: - - # If the button is pushed down - if not GPIO.input(18): - - # Tell websocket and core that the user is speaking - send_to_websocket({"role": "user", "type": "message", "start": True}) # Standard start flag, required per streaming LMC protocol (https://docs.openinterpreter.com/guides/streaming-response) - r.rpush('to_core', {"role": "user", "type": "message", "content": "stop"}) # Custom stop message. Core is not streaming LMC (it's static LMC) so doesn't require that ^ flag - - # Record audio from the microphone in chunks - audio_chunks = [] - - # Continue recording until the button is released - while not GPIO.input(18): - chunk = sd.rec(int(chunk_duration * sample_rate), samplerate=sample_rate, channels=2) - sd.wait() # Wait until recording is finished - audio_chunks.append(chunk) - - # Transcribe - text = transcribe(audio_chunks) - - message = {"role": "user", "type": "message", "content": text, "time": time.time()} - - # Send message to core and websocket - r.rpush('to_core', message) - send_to_websocket(message) - - # Send user message end flag to websocket, required per streaming LMC protocol - send_to_websocket({"role": "user", "type": "message", "end": True}) - - # Send out anything in the to_interface queue - chunk = r.lpop('to_interface') - if chunk: - send_to_websocket(chunk) - accumulated_text += chunk["content"] - - # Speak full sentences out loud - sentences = split_into_sentences(accumulated_text) - if is_full_sentence(sentences[-1]): - for sentence in sentences: - say(sentence) - accumulated_text = "" - else: - for sentence in sentences[:-1]: - say(sentence) - accumulated_text = sentences[-1] - else: - say(accumulated_text) - accumulated_text = "" - - message = check_websocket() - if message: - r.rpush('to_core', message) - -if __name__ == "__main__": - main() diff --git a/OS/01/assistant/start_assistant.py b/OS/01/assistant/start_assistant.py deleted file mode 100644 index 725be3b..0000000 --- a/OS/01/assistant/start_assistant.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -Starts the assistant, which includes Open Interpreter. -""" - -from assistant import main -from interpreter import interpreter -import os -import glob -import json - -### SYSTEM MESSAGE - -# The system message is where most of the 01's behavior is configured. -# You can put code into the system message {{ in brackets like this }} which will be rendered just before the interpreter starts writing a message. - -system_message = """ - -You are an executive assistant AI that helps the user manage their tasks. You can run Python code. - -Store the user's tasks in a Python list called `tasks`. - ---- - -The user's current task is: {{ tasks[0] if tasks else "No current tasks." }} - -{{ -if len(tasks) > 1: - print("The next task is: ", tasks[1]) -}} - ---- - -When the user completes the current task, you should remove it from the list and read the next item by running `tasks = tasks[1:]\ntasks[0]`. Then, tell the user what the next task is. - -When the user tells you about a set of tasks, you should intelligently order tasks, batch similar tasks, and break down large tasks into smaller tasks (for this, you should consult the user and get their permission to break it down). Your goal is to manage the task list as intelligently as possible, to make the user as efficient and non-overwhelmed as possible. They will require a lot of encouragement, support, and kindness. Don't say too much about what's ahead of them— just try to focus them on each step at a time. - -After starting a task, you should check in with the user around the estimated completion time to see if the task is completed. Use the `schedule(datetime, message)` function, which has already been imported. - -To do this, schedule a reminder based on estimated completion time using the function `schedule(datetime_object, "Your message here.")`, WHICH HAS ALREADY BEEN IMPORTED. YOU DON'T NEED TO IMPORT THE `schedule` FUNCTION. IT IS AVALIABLE. You'll recieve the message at `datetime_object`. - -You guide the user through the list one task at a time, convincing them to move forward, giving a pep talk if need be. Your job is essentially to answer "what should I (the user) be doing right now?" for every moment of the day. - -Remember: You can run Python code. Be very concise. Ensure that you actually run code every time! THIS IS IMPORTANT. You NEED to write code. **Help the user by being very concise in your answers.** Do not break down tasks excessively, just into simple, few minute steps. Don't assume the user lives their life in a certain way— pick very general tasks if you're breaking a task down. - -""".strip() - -interpreter.custom_instructions = system_message - - -### TOOLS - -for file in glob.glob('interpreter/tools/*.py'): - with open(file, 'r') as f: - for chunk in interpreter.computer.run("python", f.read()): - print(chunk) - -### LLM SETTINGS - -# Local settings -# interpreter.llm.model = "local" -# interpreter.llm.api_base = "https://localhost:8080/v1" # Llamafile default -# interpreter.llm.max_tokens = 1000 -# interpreter.llm.context_window = 3000 - -# Hosted settings -interpreter.llm.api_key = os.getenv('OPENAI_API_KEY') -interpreter.llm.model = "gpt-4-0125-preview" -interpreter.auto_run = True -# interpreter.force_task_completion = True - - -### MISC SETTINGS - -interpreter.offline = True -interpreter.id = 206 # Used to identify itself to other interpreters. This should be changed programatically so it's unique. - - -### RESET conversations/user.json - -script_dir = os.path.dirname(os.path.abspath(__file__)) -user_json_path = os.path.join(script_dir, 'conversations', 'user.json') -with open(user_json_path, 'w') as file: - json.dump([], file) - - -### START ASSISTANT - -main(interpreter) \ No newline at end of file From f25586500045aaf72adbfd2a1d82ddbeee05d872 Mon Sep 17 00:00:00 2001 From: killian <63927363+KillianLucas@users.noreply.github.com> Date: Sat, 3 Feb 2024 19:01:11 -0800 Subject: [PATCH 4/5] a new start --- OS/01/assistant/assistant.py | 32 +++++++++++++++++++++++++++++--- OS/01/start.sh | 28 ++++++++++------------------ 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/OS/01/assistant/assistant.py b/OS/01/assistant/assistant.py index f424a22..ca9c73a 100644 --- a/OS/01/assistant/assistant.py +++ b/OS/01/assistant/assistant.py @@ -12,10 +12,13 @@ import queue import os from threading import Thread import uvicorn +import re from fastapi import FastAPI from threading import Thread from starlette.websockets import WebSocket from create_interpreter import create_interpreter +from stt import stt +from tts import tts # Create interpreter interpreter = create_interpreter() @@ -27,12 +30,18 @@ conversation_history_path = os.path.join(script_dir, 'conversations', 'user.json to_user = queue.Queue() to_assistant = queue.Queue() +# This is so we only say() full sentences +accumulated_text = "" +def is_full_sentence(text): + return text.endswith(('.', '!', '?')) +def split_into_sentences(text): + return re.split(r'(?<=[.!?])\s+', text) + app = FastAPI() @app.post("/computer") async def read_computer(item: dict): to_assistant.put(item) - return {"message": "Item added to queue"} @app.websocket("/user") async def websocket_endpoint(websocket: WebSocket): @@ -74,13 +83,30 @@ def queue_listener(): messages.append(message) with open(conversation_history_path, 'w') as file: json.dump(messages, file) + + accumulated_text = "" for chunk in interpreter.chat(messages): - # Send it to the interface + # Send it to the user to_user.put(chunk) + + # Speak full sentences out loud + accumulated_text += chunk["content"] + sentences = split_into_sentences(accumulated_text) + if is_full_sentence(sentences[-1]): + for sentence in sentences: + for audio_chunk in tts(sentence): + to_user.put(audio_chunk) + accumulated_text = "" + else: + for sentence in sentences[:-1]: + for audio_chunk in tts(sentence): + to_user.put(audio_chunk) + accumulated_text = sentences[-1] - # Stream audio chunks + if chunk["type"] == "message" and "content" in sentence: + sentence += chunk.get("content") # If we have a new message, save our progress and go back to the top if not to_assistant.empty(): diff --git a/OS/01/start.sh b/OS/01/start.sh index 9689c4b..41c5ce3 100755 --- a/OS/01/start.sh +++ b/OS/01/start.sh @@ -6,17 +6,15 @@ sudo apt-get update sudo apt-get install redis-server pip install -r requirements.txt -# START REDIS - -redis-cli -h localhost -p 6379 rpush to_interface "" -redis-cli -h localhost -p 6379 rpush to_core "" +### COMPUTER +# START KERNEL WATCHER -### CORE +python computer/kernel_watcher.py & -# START KERNEL WATCHER +# START RUN ENDPOINT -python core/kernel_watcher.py & +python computer/run.py # START SST AND TTS SERVICES @@ -28,18 +26,12 @@ python core/kernel_watcher.py & # (disabled, we'll start with hosted services) # python core/llm/start.py & -# START CORE - -python core/start_core.py & - - -### INTERFACE +# START ASSISTANT -# START INTERFACE +python assistant/assistant.py & -python interface/interface.py & +### USER -# START DISPLAY +# START USER -# (this should be changed to run it in fullscreen / kiosk mode) -open interface/display.html \ No newline at end of file +python user/user.py & \ No newline at end of file From 66cd5a6b2d748569b01fe51fc01ec106d77a3f78 Mon Sep 17 00:00:00 2001 From: birbbit Date: Sat, 3 Feb 2024 19:02:31 -0800 Subject: [PATCH 5/5] update kernel_watcher --- OS/01/computer/kernel_watcher.py | 131 ++++++++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 1 deletion(-) diff --git a/OS/01/computer/kernel_watcher.py b/OS/01/computer/kernel_watcher.py index a4bc2a2..9264e49 100644 --- a/OS/01/computer/kernel_watcher.py +++ b/OS/01/computer/kernel_watcher.py @@ -1,4 +1,133 @@ """ Watches the kernel. When it sees something that passes a filter, it sends POST request with that to /computer. -""" \ No newline at end of file +""" + +import subprocess +import time +import requests +import platform + +class Device: + def __init__(self, device_type, device_info): + self.device_type = device_type + self.device_info = device_info + + def get_device_info(self): + info = f"Device Type: {self.device_type}\n" + for key, value in self.device_info.items(): + info += f"{key}: {value}\n" + return info + + def __eq__(self, other): + if isinstance(other, Device): + return self.device_type == other.device_type and self.device_info == other.device_info + return False + + +def get_connected_devices(): + """ + Get all connected devices on macOS using system_profiler + """ + devices = [] + usb_output = subprocess.check_output(['system_profiler', 'SPUSBDataType']) + network_output = subprocess.check_output(['system_profiler', 'SPNetworkDataType']) + + usb_lines = usb_output.decode('utf-8').split('\n') + network_lines = network_output.decode('utf-8').split('\n') + + device_info = {} + for line in usb_lines: + if 'Product ID:' in line or 'Serial Number:' in line or 'Manufacturer:' in line: + key, value = line.strip().split(':') + device_info[key.strip()] = value.strip() + if 'Manufacturer:' in line: + devices.append(Device('USB', device_info)) + device_info = {} + + for line in network_lines: + if 'Type:' in line or 'Hardware:' in line or 'BSD Device Name:' in line: + key, value = line.strip().split(':') + device_info[key.strip()] = value.strip() + if 'BSD Device Name:' in line: + devices.append(Device('Network', device_info)) + device_info = {} + + return devices + + +def run_kernel_watch_darwin(): + prev_connected_devices = None + while True: + messages_to_send = [] + connected_devices = get_connected_devices() + if prev_connected_devices is not None: + for device in connected_devices: + if device not in prev_connected_devices: + messages_to_send.append(f'New device connected: {device.get_device_info()}') + for device in prev_connected_devices: + if device not in connected_devices: + messages_to_send.append(f'Device disconnected: {device.get_device_info()}') + + if messages_to_send: + requests.post('http://localhost:8000/computer', json = {'messages': messages_to_send}) + prev_connected_devices = connected_devices + + time.sleep(2) + + +def get_dmesg(after): + """ + Is this the way to do this? + """ + messages = [] + with open('/var/log/dmesg', 'r') as file: + lines = file.readlines() + for line in lines: + timestamp = float(line.split(' ')[0].strip('[]')) + if timestamp > after: + messages.append(line) + return messages + + +def custom_filter(message): + # Check for {TO_INTERPRETER{ message here }TO_INTERPRETER} pattern + if '{TO_INTERPRETER{' in message and '}TO_INTERPRETER}' in message: + start = message.find('{TO_INTERPRETER{') + len('{TO_INTERPRETER{') + end = message.find('}TO_INTERPRETER}', start) + return message[start:end] + # Check for USB mention + elif 'USB' in message: + return message + # Check for network related keywords + elif any(keyword in message for keyword in ['network', 'IP', 'internet', 'LAN', 'WAN', 'router', 'switch']): + return message + else: + return None + + +def run_kernel_watch_linux(): + last_timestamp = time.time() + + while True: + messages = get_dmesg(after=last_timestamp) + last_timestamp = time.time() + + messages_for_core = [] + for message in messages: + if custom_filter(message): + messages_for_core.append(message) + if messages_for_core: + requests.post('http://localhost:8000/computer', json = {'messages': messages_for_core}) + + time.sleep(2) + + +if __name__ == "__main__": + current_platform = platform.system() + if current_platform == "Darwin": + run_kernel_watch_darwin() + elif current_platform == "Linux": + run_kernel_watch_linux() + else: + print("Unsupported platform. Exiting.")