From 4b25239d0f4dbc3c634d7d0b28506f1d44107a45 Mon Sep 17 00:00:00 2001 From: Ben Xu Date: Mon, 17 Jun 2024 15:11:31 -0700 Subject: [PATCH 1/3] stash server changes --- software/source/clients/base_device.py | 89 +++++++----- software/source/server/async_interpreter.py | 19 +-- software/source/server/async_server.py | 145 +++++++++++--------- 3 files changed, 146 insertions(+), 107 deletions(-) diff --git a/software/source/clients/base_device.py b/software/source/clients/base_device.py index b713601..767f4b0 100644 --- a/software/source/clients/base_device.py +++ b/software/source/clients/base_device.py @@ -2,6 +2,7 @@ from dotenv import load_dotenv load_dotenv() # take environment variables from .env. +import requests import subprocess import os import sys @@ -12,6 +13,7 @@ from pynput import keyboard import json import traceback import websockets +import websockets.sync.client import queue from pydub import AudioSegment from pydub.playback import play @@ -169,11 +171,11 @@ class Device: elapsed_time = time.time() - self.playback_latency print(f"Time from request to playback: {elapsed_time} seconds") self.playback_latency = None - + """ if audio is not None: mpv_process.stdin.write(audio) # type: ignore mpv_process.stdin.flush() # type: ignore - """ + args = ["ffplay", "-autoexit", "-", "-nodisp"] proc = subprocess.Popen( args=args, @@ -183,9 +185,8 @@ class Device: ) out, err = proc.communicate(input=audio) proc.poll() - - play(audio) """ + play(audio) # self.audiosegments.remove(audio) # await asyncio.sleep(0.1) except asyncio.exceptions.CancelledError: @@ -361,7 +362,7 @@ class Device: async def websocket_communication(self, WS_URL): print("websocket communication was called!!!!") show_connection_log = True - + """ async def exec_ws_communication(websocket): if CAMERA_ENABLED: print( @@ -373,11 +374,11 @@ class Device: asyncio.create_task(self.message_sender(websocket)) while True: - await asyncio.sleep(0.0001) + await asyncio.sleep(0) chunk = await websocket.recv() - logger.debug(f"Got this message from the server: {type(chunk)} {chunk}") - # print((f"Got this message from the server: {type(chunk)} {chunk}")) + #logger.debug(f"Got this message from the server: {type(chunk)} {chunk}") + print((f"Got this message from the server: {type(chunk)}")) if type(chunk) == str: chunk = json.loads(chunk) @@ -388,7 +389,7 @@ class Device: continue # At this point, we have our message - # print("checkpoint reached!", message) + print("checkpoint reached!") if isinstance(message, bytes): # if message["type"] == "audio" and message["format"].startswith("bytes"): @@ -398,23 +399,23 @@ class Device: audio_bytes = message # Create an AudioSegment instance with the raw data - """ + audio = AudioSegment( # raw audio data (bytes) data=audio_bytes, # signed 16-bit little-endian format sample_width=2, # 24,000 Hz frame rate - frame_rate=16000, + frame_rate=24000, # mono sound channels=1, ) - """ - # print("audio segment was created") - await self.audiosegments.put(audio_bytes) - # await self.audiosegments.put(audio) + print("audio segment was created") + #await self.audiosegments.put(audio_bytes) + + await self.audiosegments.put(audio) # Run the code if that's the client's job if os.getenv("CODE_RUNNER") == "client": @@ -424,42 +425,65 @@ class Device: result = interpreter.computer.run(language, code) send_queue.put(result) + """ if is_win10(): logger.info("Windows 10 detected") # Workaround for Windows 10 not latching to the websocket server. # See https://github.com/OpenInterpreter/01/issues/197 try: ws = websockets.connect(WS_URL) - await exec_ws_communication(ws) + # await exec_ws_communication(ws) except Exception as e: logger.error(f"Error while attempting to connect: {e}") else: print("websocket url is", WS_URL) - while True: + i = 0 + # while True: + # try: + # i += 1 + # print("i is", i) + + # # Hit the /ping endpoint + # ping_url = f"http://{self.server_url}/ping" + # response = requests.get(ping_url) + # print(response.text) + # # async with aiohttp.ClientSession() as session: + # # async with session.get(ping_url) as response: + # # print(f"Ping response: {await response.text()}") + + for i in range(3): + print(i) try: async with websockets.connect(WS_URL) as websocket: - print("awaiting exec_ws_communication") - await exec_ws_communication(websocket) + print("happi happi happi :DDDDDDDDDDDDD") + # await exec_ws_communication(websocket) + # print("exiting exec_ws_communication") except: - logger.info(traceback.format_exc()) - if show_connection_log: - logger.info(f"Connecting to `{WS_URL}`...") - show_connection_log = False - await asyncio.sleep(2) + print("exception in websocket communication!!!!!!!!!!!!!!!!!") + traceback.print_exc() + + # except: + # print("exception in websocket communication!!!!!!!!!!!!!!!!!") + # traceback.print_exc() + # if show_connection_log: + # logger.info(f"Connecting to `{WS_URL}`...") + # show_connection_log = False + # await asyncio.sleep(2) async def start_async(self): print("start async was called!!!!!") # Configuration for WebSocket - WS_URL = f"ws://{self.server_url}" + WS_URL = f"ws://{self.server_url}/" # Start the WebSocket communication - asyncio.create_task(self.websocket_communication(WS_URL)) + await self.websocket_communication(WS_URL) + """ # Start watching the kernel if it's your job to do that if os.getenv("CODE_RUNNER") == "client": asyncio.create_task(put_kernel_messages_into_queue(send_queue)) - asyncio.create_task(self.play_audiosegments()) + #asyncio.create_task(self.play_audiosegments()) # If Raspberry Pi, add the button listener, otherwise use the spacebar if current_platform.startswith("raspberry-pi"): @@ -483,12 +507,11 @@ class Device: else: break else: - # Keyboard listener for spacebar press/release - listener = keyboard.Listener( - on_press=self.on_press, on_release=self.on_release - ) - listener.start() - print("listener for keyboard started!!!!!") + """ + # Keyboard listener for spacebar press/release + # listener = keyboard.Listener(on_press=self.on_press, on_release=self.on_release) + # listener.start() + # print("listener for keyboard started!!!!!") def start(self): print("device was started!!!!!!") diff --git a/software/source/server/async_interpreter.py b/software/source/server/async_interpreter.py index 209ff73..1e6473a 100644 --- a/software/source/server/async_interpreter.py +++ b/software/source/server/async_interpreter.py @@ -38,7 +38,7 @@ class AsyncInterpreter: self.stt = AudioToTextRecorder( model="tiny.en", spinner=False, use_microphone=False ) - self.stt.stop() # It needs this for some reason + self.stt.stop() # TTS if self.interpreter.tts == "coqui": @@ -118,8 +118,6 @@ class AsyncInterpreter: """ self.interpreter.messages = self.active_chat_messages - # self.beeper.start() - self.stt.stop() # message = self.stt.text() # print("THE MESSAGE:", message) @@ -137,15 +135,9 @@ class AsyncInterpreter: self.stt_latency = end_stt - start_stt print("STT LATENCY", self.stt_latency) - # print(message) - end_interpreter = 0 - - # print(message) def generate(message): last_lmc_start_flag = self._last_lmc_start_flag self.interpreter.messages = self.active_chat_messages - # print("πŸ€πŸ€πŸ€πŸ€GENERATING, using these messages: ", self.interpreter.messages) - # print("πŸ€ πŸ€ πŸ€ πŸ€ active_chat_messages: ", self.active_chat_messages) print("message is", message) for chunk in self.interpreter.chat(message, display=True, stream=True): @@ -209,7 +201,7 @@ class AsyncInterpreter: text_iterator = generate(message) self.tts.feed(text_iterator) - self.tts.play_async(on_audio_chunk=self.on_tts_chunk, muted=True) + self.tts.play_async(on_audio_chunk=self.on_tts_chunk, muted=False) while True: if self.tts.is_playing(): @@ -236,7 +228,7 @@ class AsyncInterpreter: break async def _on_tts_chunk_async(self, chunk): - # print("SENDING TTS CHUNK") + print(f"Adding chunk to output queue") await self._add_to_queue(self._output_queue, chunk) def on_tts_chunk(self, chunk): @@ -244,4 +236,7 @@ class AsyncInterpreter: asyncio.run(self._on_tts_chunk_async(chunk)) async def output(self): - return await self._output_queue.get() + print("entering output method") + value = await self._output_queue.get() + print("output method returning") + return value diff --git a/software/source/server/async_server.py b/software/source/server/async_server.py index ace4b4a..f4fcc3e 100644 --- a/software/source/server/async_server.py +++ b/software/source/server/async_server.py @@ -5,6 +5,7 @@ from fastapi import FastAPI, WebSocket, Header from fastapi.responses import PlainTextResponse from uvicorn import Config, Server from interpreter import interpreter as base_interpreter +from starlette.websockets import WebSocketDisconnect from .async_interpreter import AsyncInterpreter from fastapi.middleware.cors import CORSMiddleware from typing import List, Dict, Any @@ -23,18 +24,11 @@ base_interpreter.llm.model = "groq/llama3-8b-8192" base_interpreter.llm.api_key = os.environ["GROQ_API_KEY"] base_interpreter.llm.supports_functions = False base_interpreter.auto_run = True +base_interpreter.tts = "elevenlabs" os.environ["STT_RUNNER"] = "server" os.environ["TTS_RUNNER"] = "server" -# Parse command line arguments for port number -""" -parser = argparse.ArgumentParser(description="FastAPI server.") -parser.add_argument("--port", type=int, default=8000, help="Port to run on.") -args = parser.parse_args() -""" -base_interpreter.tts = "coqui" - async def main(server_host, server_port): interpreter = AsyncInterpreter(base_interpreter) @@ -60,84 +54,111 @@ async def main(server_host, server_port): print("πŸͺΌπŸͺΌπŸͺΌπŸͺΌπŸͺΌπŸͺΌ Messages loaded: ", interpreter.active_chat_messages) return {"status": "success"} + print("About to set up the websocker endpoint!!!!!!!!!!!!!!!!!!!!!!!!!") + @app.websocket("/") async def websocket_endpoint(websocket: WebSocket): + print("websocket hit") await websocket.accept() - try: + print("websocket accepted") - async def receive_input(): + async def send_output(): + try: while True: - if websocket.client_state == "DISCONNECTED": - break + output = await interpreter.output() + + if isinstance(output, bytes): + print("server sending bytes output") + try: + await websocket.send_bytes(output) + print("server successfully sent bytes output") + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + elif isinstance(output, dict): + print("server sending text output") + try: + await websocket.send_text(json.dumps(output)) + print("server successfully sent text output") + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + except asyncio.CancelledError: + print("WebSocket connection closed") + traceback.print_exc() + + async def receive_input(): + try: + while True: + print("server awaiting input") data = await websocket.receive() if isinstance(data, bytes): - await interpreter.input(data) + try: + await interpreter.input(data) + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + elif "bytes" in data: - await interpreter.input(data["bytes"]) - # print("RECEIVED INPUT", data) + try: + await interpreter.input(data["bytes"]) + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + elif "text" in data: - # print("RECEIVED INPUT", data) - await interpreter.input(data["text"]) + try: + await interpreter.input(data["text"]) + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + except asyncio.CancelledError: + print("WebSocket connection closed") + traceback.print_exc() - async def send_output(): - while True: - output = await interpreter.output() + try: + send_task = asyncio.create_task(send_output()) + receive_task = asyncio.create_task(receive_input()) + + print("server starting to handle ws connection") + """ + done, pending = await asyncio.wait( + [send_task, receive_task], + return_when=asyncio.FIRST_COMPLETED, + ) - if isinstance(output, bytes): - # print(f"Sending {len(output)} bytes of audio data.") - await websocket.send_bytes(output) - # we dont send out bytes rn, no TTS + for task in pending: + task.cancel() - elif isinstance(output, dict): - # print("sending text") - await websocket.send_text(json.dumps(output)) + for task in done: + if task.exception() is not None: + raise + """ + await asyncio.gather(send_task, receive_task) + + print("server finished handling ws connection") - await asyncio.gather(send_output(), receive_input()) + except WebSocketDisconnect: + print("WebSocket disconnected") except Exception as e: print(f"WebSocket connection closed with exception: {e}") traceback.print_exc() finally: - if not websocket.client_state == "DISCONNECTED": - await websocket.close() + print("server closing ws connection") + await websocket.close() print(f"Starting server on {server_host}:{server_port}") config = Config(app, host=server_host, port=server_port, lifespan="on") server = Server(config) await server.serve() - class Rename(BaseModel): - input: str - - @app.post("/rename-chat") - async def rename_chat(body_content: Rename, x_api_key: str = Header(None)): - print("RENAME CHAT REQUEST in PY πŸŒ™πŸŒ™πŸŒ™πŸŒ™") - input_value = body_content.input - client = OpenAI( - # defaults to os.environ.get("OPENAI_API_KEY") - api_key=x_api_key, - ) - try: - response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": f"Given the following chat snippet, create a unique and descriptive title in less than 8 words. Your answer must not be related to customer service.\n\n{input_value}", - } - ], - temperature=0.3, - stream=False, - ) - print(response) - completion = response["choices"][0]["message"]["content"] - return {"data": {"content": completion}} - except Exception as e: - print(f"Error: {e}") - traceback.print_exc() - return {"error": str(e)} - if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main("localhost", 8000)) From d8d57f3f1dad9e071327157f071ac784e547307e Mon Sep 17 00:00:00 2001 From: Ben Xu Date: Tue, 18 Jun 2024 08:36:32 -0700 Subject: [PATCH 2/3] add plyer pywinctl --- software/poetry.lock | 98 ++++++++++++++++++++++++++++++++++++++++- software/pyproject.toml | 2 + 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/software/poetry.lock b/software/poetry.lock index 45f3b5c..ec4afe5 100644 --- a/software/poetry.lock +++ b/software/poetry.lock @@ -1286,6 +1286,23 @@ files = [ {file = "evdev-1.7.1.tar.gz", hash = "sha256:0c72c370bda29d857e188d931019c32651a9c1ea977c08c8d939b1ced1637fde"}, ] +[[package]] +name = "ewmhlib" +version = "0.2" +description = "Extended Window Manager Hints implementation in Python 3" +optional = false +python-versions = "*" +files = [ + {file = "EWMHlib-0.2-py3-none-any.whl", hash = "sha256:f5b07d8cfd4c7734462ee744c32d490f2f3233fa7ab354240069344208d2f6f5"}, +] + +[package.dependencies] +python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} +typing-extensions = ">=4.4.0" + +[package.extras] +dev = ["mypy (>=0.990)", "types-python-xlib (>=0.32)", "types-setuptools (>=65.5)"] + [[package]] name = "exceptiongroup" version = "1.2.1" @@ -3781,6 +3798,23 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "plyer" +version = "2.1.0" +description = "Platform-independent wrapper for platform-dependent APIs" +optional = false +python-versions = "*" +files = [ + {file = "plyer-2.1.0-py2.py3-none-any.whl", hash = "sha256:1b1772060df8b3045ed4f08231690ec8f7de30f5a004aa1724665a9074eed113"}, + {file = "plyer-2.1.0.tar.gz", hash = "sha256:65b7dfb7e11e07af37a8487eb2aa69524276ef70dad500b07228ce64736baa61"}, +] + +[package.extras] +android = ["pyjnius"] +dev = ["flake8", "mock"] +ios = ["pyobjus"] +macosx = ["pyobjus"] + [[package]] name = "pooch" version = "1.8.1" @@ -4197,6 +4231,26 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pymonctl" +version = "0.92" +description = "Cross-Platform toolkit to get info on and control monitors connected" +optional = false +python-versions = "*" +files = [ + {file = "PyMonCtl-0.92-py3-none-any.whl", hash = "sha256:2495d8dab78f9a7dbce37e74543e60b8bd404a35c3108935697dda7768611b5a"}, +] + +[package.dependencies] +ewmhlib = {version = ">=0.1", markers = "sys_platform == \"linux\""} +pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""} +python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} +pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""} +typing-extensions = ">=4.4.0" + +[package.extras] +dev = ["mypy (>=0.990)", "pywinctl (>=0.3)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"] + [[package]] name = "pymsgbox" version = "1.0.9" @@ -7228,6 +7282,48 @@ files = [ {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, ] +[[package]] +name = "pywinbox" +version = "0.7" +description = "Cross-Platform and multi-monitor toolkit to handle rectangular areas and windows box" +optional = false +python-versions = "*" +files = [ + {file = "PyWinBox-0.7-py3-none-any.whl", hash = "sha256:8b2506a8dd7afa0a910b368762adfac885274132ef9151b0c81b0d2c6ffd6f83"}, +] + +[package.dependencies] +ewmhlib = {version = ">=0.1", markers = "sys_platform == \"linux\""} +pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""} +python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} +pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""} +typing-extensions = ">=4.4.0" + +[package.extras] +dev = ["mypy (>=0.990)", "pywinctl (>=0.3)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"] + +[[package]] +name = "pywinctl" +version = "0.4" +description = "Cross-Platform toolkit to get info on and control windows on screen" +optional = false +python-versions = "*" +files = [ + {file = "PyWinCtl-0.4-py3-none-any.whl", hash = "sha256:8c4a92bd57e35fd280c5c04f048cc822e236abffe2fa17351096b0e28907172d"}, +] + +[package.dependencies] +ewmhlib = {version = ">=0.2", markers = "sys_platform == \"linux\""} +pymonctl = ">=0.92" +pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""} +python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} +pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""} +pywinbox = ">=0.7" +typing-extensions = ">=4.4.0" + +[package.extras] +dev = ["mypy (>=0.990)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"] + [[package]] name = "pyyaml" version = "6.0.1" @@ -9454,4 +9550,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "de07d24f74d39335d2c89d83cbe681bacbeaeffdb30227cb9f6955dc4614dfca" +content-hash = "0843bdc2785fb5bbabef0ed3f92bf574c9fc20ddf2b16edc4a00d38e32a8fa94" diff --git a/software/pyproject.toml b/software/pyproject.toml index ab54d5c..2869a0b 100644 --- a/software/pyproject.toml +++ b/software/pyproject.toml @@ -47,6 +47,8 @@ pywebview = "*" pyobjc = "*" sentry-sdk = "^2.4.0" +plyer = "^2.1.0" +pywinctl = "^0.4" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From 3011e5535e76aeb59e097982e73b0848ad207e2a Mon Sep 17 00:00:00 2001 From: Ben Xu Date: Tue, 18 Jun 2024 08:50:50 -0700 Subject: [PATCH 3/3] resolve dateparser dependencies --- software/poetry.lock | 178 ++++++++++++++++++++++++++++++++++++++-- software/pyproject.toml | 5 +- 2 files changed, 175 insertions(+), 8 deletions(-) diff --git a/software/poetry.lock b/software/poetry.lock index ec4afe5..5fdbce9 100644 --- a/software/poetry.lock +++ b/software/poetry.lock @@ -2173,6 +2173,27 @@ qtconsole = ["qtconsole"] test = ["pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath"] test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath", "trio"] +[[package]] +name = "ipywidgets" +version = "8.1.3" +description = "Jupyter interactive widgets" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ipywidgets-8.1.3-py3-none-any.whl", hash = "sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2"}, + {file = "ipywidgets-8.1.3.tar.gz", hash = "sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c"}, +] + +[package.dependencies] +comm = ">=0.1.3" +ipython = ">=6.1.0" +jupyterlab-widgets = ">=3.0.11,<3.1.0" +traitlets = ">=4.3.1" +widgetsnbextension = ">=4.0.11,<4.1.0" + +[package.extras] +test = ["ipykernel", "jsonschema", "pytest (>=3.6.0)", "pytest-cov", "pytz"] + [[package]] name = "isort" version = "5.13.2" @@ -2305,6 +2326,17 @@ traitlets = ">=5.3" docs = ["myst-parser", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] test = ["ipykernel", "pre-commit", "pytest (<8)", "pytest-cov", "pytest-timeout"] +[[package]] +name = "jupyterlab-widgets" +version = "3.0.11" +description = "Jupyter interactive widgets for JupyterLab" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jupyterlab_widgets-3.0.11-py3-none-any.whl", hash = "sha256:78287fd86d20744ace330a61625024cf5521e1c012a352ddc0a3cdc2348becd0"}, + {file = "jupyterlab_widgets-3.0.11.tar.gz", hash = "sha256:dd5ac679593c969af29c9bed054c24f26842baa51352114736756bc035deee27"}, +] + [[package]] name = "keyboard" version = "0.13.5" @@ -3558,22 +3590,32 @@ git-python = ">=1.0.3,<2.0.0" html2image = ">=2.0.4.3,<3.0.0.0" inquirer = ">=3.1.3,<4.0.0" ipykernel = ">=6.26.0,<7.0.0" +ipywidgets = {version = ">=8.1.2,<9.0.0", optional = true, markers = "extra == \"os\""} jupyter-client = ">=8.6.0,<9.0.0" litellm = ">=1.35.32,<2.0.0" matplotlib = ">=3.8.2,<4.0.0" nltk = ">=3.8.1,<4.0.0" +opencv-python = {version = ">=4.8.1.78,<5.0.0.0", optional = true, markers = "extra == \"os\" or extra == \"local\""} platformdirs = ">=4.2.0,<5.0.0" +plyer = {version = ">=2.1.0,<3.0.0", optional = true, markers = "extra == \"os\""} psutil = ">=5.9.6,<6.0.0" +pyautogui = {version = ">=0.9.54,<0.10.0", optional = true, markers = "extra == \"os\""} pydantic = ">=2.6.4,<3.0.0" pyreadline3 = {version = ">=3.4.1,<4.0.0", markers = "sys_platform == \"win32\""} +pytesseract = {version = ">=0.3.10,<0.4.0", optional = true, markers = "extra == \"os\" or extra == \"local\""} +pywinctl = {version = ">=0.3,<0.4", optional = true, markers = "extra == \"os\""} pyyaml = ">=6.0.1,<7.0.0" rich = ">=13.4.2,<14.0.0" +screeninfo = {version = ">=0.8.1,<0.9.0", optional = true, markers = "extra == \"os\""} send2trash = ">=1.8.2,<2.0.0" +sentence-transformers = {version = ">=2.5.1,<3.0.0", optional = true, markers = "extra == \"os\""} setuptools = "*" six = ">=1.16.0,<2.0.0" tiktoken = ">=0.6.0,<0.7.0" +timm = {version = ">=0.9.16,<0.10.0", optional = true, markers = "extra == \"os\""} tokentrim = ">=0.1.13,<0.2.0" toml = ">=0.10.2,<0.11.0" +torch = {version = ">=2.2.1,<3.0.0", optional = true, markers = "extra == \"os\" or extra == \"local\""} wget = ">=3.2,<4.0" [package.extras] @@ -7017,6 +7059,21 @@ Pillow = [ {version = ">=9.3.0", markers = "python_version == \"3.11\""}, ] +[[package]] +name = "pytesseract" +version = "0.3.10" +description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytesseract-0.3.10-py3-none-any.whl", hash = "sha256:8f22cc98f765bf13517ead0c70effedb46c153540d25783e04014f28b55a5fc6"}, + {file = "pytesseract-0.3.10.tar.gz", hash = "sha256:f1c3a8b0f07fd01a1085d451f5b8315be6eec1d5577a6796d46dc7a62bd4120f"}, +] + +[package.dependencies] +packaging = ">=21.3" +Pillow = ">=8.0.0" + [[package]] name = "pytest" version = "8.2.1" @@ -7304,21 +7361,20 @@ dev = ["mypy (>=0.990)", "pywinctl (>=0.3)", "types-python-xlib (>=0.32)", "type [[package]] name = "pywinctl" -version = "0.4" +version = "0.3" description = "Cross-Platform toolkit to get info on and control windows on screen" optional = false python-versions = "*" files = [ - {file = "PyWinCtl-0.4-py3-none-any.whl", hash = "sha256:8c4a92bd57e35fd280c5c04f048cc822e236abffe2fa17351096b0e28907172d"}, + {file = "PyWinCtl-0.3-py3-none-any.whl", hash = "sha256:3603981c87b0c64987e7be857d89450f98792b01f49006a17dac758e11141dd7"}, ] [package.dependencies] -ewmhlib = {version = ">=0.2", markers = "sys_platform == \"linux\""} -pymonctl = ">=0.92" +pymonctl = ">=0.6" pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""} python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""} -pywinbox = ">=0.7" +pywinbox = ">=0.6" typing-extensions = ">=4.4.0" [package.extras] @@ -7921,6 +7977,21 @@ dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pyde doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +[[package]] +name = "screeninfo" +version = "0.8.1" +description = "Fetch location and size of physical screens." +optional = false +python-versions = ">=3.6.2,<4.0.0" +files = [ + {file = "screeninfo-0.8.1-py3-none-any.whl", hash = "sha256:e97d6b173856edcfa3bd282f81deb528188aff14b11ec3e195584e7641be733c"}, + {file = "screeninfo-0.8.1.tar.gz", hash = "sha256:9983076bcc7e34402a1a9e4d7dabf3729411fd2abb3f3b4be7eba73519cd2ed1"}, +] + +[package.dependencies] +Cython = {version = "*", markers = "sys_platform == \"darwin\""} +pyobjc-framework-Cocoa = {version = "*", markers = "sys_platform == \"darwin\""} + [[package]] name = "send2trash" version = "1.8.3" @@ -7937,6 +8008,30 @@ nativelib = ["pyobjc-framework-Cocoa", "pywin32"] objc = ["pyobjc-framework-Cocoa"] win32 = ["pywin32"] +[[package]] +name = "sentence-transformers" +version = "2.7.0" +description = "Multilingual text embeddings" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "sentence_transformers-2.7.0-py3-none-any.whl", hash = "sha256:6a7276b05a95931581bbfa4ba49d780b2cf6904fa4a171ec7fd66c343f761c98"}, + {file = "sentence_transformers-2.7.0.tar.gz", hash = "sha256:2f7df99d1c021dded471ed2d079e9d1e4fc8e30ecb06f957be060511b36f24ea"}, +] + +[package.dependencies] +huggingface-hub = ">=0.15.1" +numpy = "*" +Pillow = "*" +scikit-learn = "*" +scipy = "*" +torch = ">=1.11.0" +tqdm = "*" +transformers = ">=4.34.0,<5.0.0" + +[package.extras] +dev = ["pre-commit", "pytest", "ruff (>=0.3.0)"] + [[package]] name = "sentry-sdk" version = "2.4.0" @@ -8657,6 +8752,24 @@ requests = ">=2.26.0" [package.extras] blobfile = ["blobfile (>=2)"] +[[package]] +name = "timm" +version = "0.9.16" +description = "PyTorch Image Models" +optional = false +python-versions = ">=3.8" +files = [ + {file = "timm-0.9.16-py3-none-any.whl", hash = "sha256:bf5704014476ab011589d3c14172ee4c901fd18f9110a928019cac5be2945914"}, + {file = "timm-0.9.16.tar.gz", hash = "sha256:891e54f375d55adf31a71ab0c117761f0e472f9f3971858ecdd1e7376b7071e6"}, +] + +[package.dependencies] +huggingface_hub = "*" +pyyaml = "*" +safetensors = "*" +torch = "*" +torchvision = "*" + [[package]] name = "tokenizers" version = "0.15.2" @@ -8915,6 +9028,48 @@ files = [ [package.dependencies] torch = "2.2.2" +[[package]] +name = "torchvision" +version = "0.17.2" +description = "image and video datasets and models for torch deep learning" +optional = false +python-versions = ">=3.8" +files = [ + {file = "torchvision-0.17.2-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:1f2910fe3c21ad6875b2720d46fad835b2e4b336e9553d31ca364d24c90b1d4f"}, + {file = "torchvision-0.17.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecc1c503fa8a54fbab777e06a7c228032b8ab78efebf35b28bc8f22f544f51f1"}, + {file = "torchvision-0.17.2-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:f400145fc108833e7c2fc28486a04989ca742146d7a2a2cc48878ebbb40cdbbd"}, + {file = "torchvision-0.17.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:e9e4bed404af33dfc92eecc2b513d21ddc4c242a7fd8708b3b09d3a26aa6f444"}, + {file = "torchvision-0.17.2-cp310-cp310-win_amd64.whl", hash = "sha256:ba2e62f233eab3d42b648c122a3a29c47cc108ca314dfd5cbb59cd3a143fd623"}, + {file = "torchvision-0.17.2-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:9b83e55ee7d0a1704f52b9c0ac87388e7a6d1d98a6bde7b0b35f9ab54d7bda54"}, + {file = "torchvision-0.17.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e031004a1bc432c980a7bd642f6c189a3efc316e423fc30b5569837166a4e28d"}, + {file = "torchvision-0.17.2-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:3bbc24b7713e8f22766992562547d8b4b10001208d372fe599255af84bfd1a69"}, + {file = "torchvision-0.17.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:833fd2e4216ced924c8aca0525733fe727f9a1af66dfad7c5be7257e97c39678"}, + {file = "torchvision-0.17.2-cp311-cp311-win_amd64.whl", hash = "sha256:6835897df852fad1015e6a106c167c83848114cbcc7d86112384a973404e4431"}, + {file = "torchvision-0.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14fd1d4a033c325bdba2d03a69c3450cab6d3a625f85cc375781d9237ca5d04d"}, + {file = "torchvision-0.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9c3acbebbe379af112b62b535820174277b1f3eed30df264a4e458d58ee4e5b2"}, + {file = "torchvision-0.17.2-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:77d680adf6ce367166a186d2c7fda3a73807ab9a03b2c31a03fa8812c8c5335b"}, + {file = "torchvision-0.17.2-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:f1c9ab3152cfb27f83aca072cac93a3a4c4e4ab0261cf0f2d516b9868a4e96f3"}, + {file = "torchvision-0.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:3f784381419f3ed3f2ec2aa42fb4aeec5bf4135e298d1631e41c926e6f1a0dff"}, + {file = "torchvision-0.17.2-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:b83aac8d78f48981146d582168d75b6c947cfb0a7693f76e219f1926f6e595a3"}, + {file = "torchvision-0.17.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1ece40557e122d79975860a005aa7e2a9e2e6c350a03e78a00ec1450083312fd"}, + {file = "torchvision-0.17.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:32dbeba3987e20f2dc1bce8d1504139fff582898346dfe8ad98d649f97ca78fa"}, + {file = "torchvision-0.17.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:35ba5c1600c3203549d2316422a659bd20c0cfda1b6085eec94fb9f35f55ca43"}, + {file = "torchvision-0.17.2-cp38-cp38-win_amd64.whl", hash = "sha256:2f69570f50b1d195e51bc03feffb7b7728207bc36efcfb1f0813712b2379d881"}, + {file = "torchvision-0.17.2-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:4868bbfa55758c8107e69a0e7dd5e77b89056035cd38b767ad5b98cdb71c0f0d"}, + {file = "torchvision-0.17.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:efd6d0dd0668e15d01a2cffadc74068433b32cbcf5692e0c4aa15fc5cb250ce7"}, + {file = "torchvision-0.17.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7dc85b397f6c6d9ef12716ce0d6e11ac2b803f5cccff6fe3966db248e7774478"}, + {file = "torchvision-0.17.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d506854c5acd69b20a8b6641f01fe841685a21c5406b56813184f1c9fc94279e"}, + {file = "torchvision-0.17.2-cp39-cp39-win_amd64.whl", hash = "sha256:067095e87a020a7a251ac1d38483aa591c5ccb81e815527c54db88a982fc9267"}, +] + +[package.dependencies] +numpy = "*" +pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" +torch = "2.2.2" + +[package.extras] +scipy = ["scipy"] + [[package]] name = "tornado" version = "6.4" @@ -9418,6 +9573,17 @@ files = [ {file = "wget-3.2.zip", hash = "sha256:35e630eca2aa50ce998b9b1a127bb26b30dfee573702782aa982f875e3f16061"}, ] +[[package]] +name = "widgetsnbextension" +version = "4.0.11" +description = "Jupyter interactive widgets for Jupyter Notebook" +optional = false +python-versions = ">=3.7" +files = [ + {file = "widgetsnbextension-4.0.11-py3-none-any.whl", hash = "sha256:55d4d6949d100e0d08b94948a42efc3ed6dfdc0e9468b2c4b128c9a2ce3a7a36"}, + {file = "widgetsnbextension-4.0.11.tar.gz", hash = "sha256:8b22a8f1910bfd188e596fe7fc05dcbd87e810c8a4ba010bdb3da86637398474"}, +] + [[package]] name = "xmod" version = "1.8.1" @@ -9550,4 +9716,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "0843bdc2785fb5bbabef0ed3f92bf574c9fc20ddf2b16edc4a00d38e32a8fa94" +content-hash = "e6b8e346d63f867e76a757bb39c2db877d51b07b549642a409685ed4aa1d0bbe" diff --git a/software/pyproject.toml b/software/pyproject.toml index 2869a0b..379769f 100644 --- a/software/pyproject.toml +++ b/software/pyproject.toml @@ -40,7 +40,7 @@ ctranslate2 = "4.1.0" py3-tts = "^3.5" elevenlabs = "1.2.2" groq = "^0.5.0" -open-interpreter = "^0.2.6" +open-interpreter = {extras = ["os"], version = "^0.2.6"} litellm = "1.35.35" openai = "1.30.5" pywebview = "*" @@ -48,7 +48,8 @@ pyobjc = "*" sentry-sdk = "^2.4.0" plyer = "^2.1.0" -pywinctl = "^0.4" +pywinctl = "^0.3" +dateparser = "^1.1.0" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api"