diff --git a/software/poetry.lock b/software/poetry.lock index 45f3b5c..5fdbce9 100644 --- a/software/poetry.lock +++ b/software/poetry.lock @@ -1286,6 +1286,23 @@ files = [ {file = "evdev-1.7.1.tar.gz", hash = "sha256:0c72c370bda29d857e188d931019c32651a9c1ea977c08c8d939b1ced1637fde"}, ] +[[package]] +name = "ewmhlib" +version = "0.2" +description = "Extended Window Manager Hints implementation in Python 3" +optional = false +python-versions = "*" +files = [ + {file = "EWMHlib-0.2-py3-none-any.whl", hash = "sha256:f5b07d8cfd4c7734462ee744c32d490f2f3233fa7ab354240069344208d2f6f5"}, +] + +[package.dependencies] +python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} +typing-extensions = ">=4.4.0" + +[package.extras] +dev = ["mypy (>=0.990)", "types-python-xlib (>=0.32)", "types-setuptools (>=65.5)"] + [[package]] name = "exceptiongroup" version = "1.2.1" @@ -2156,6 +2173,27 @@ qtconsole = ["qtconsole"] test = ["pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath"] test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath", "trio"] +[[package]] +name = "ipywidgets" +version = "8.1.3" +description = "Jupyter interactive widgets" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ipywidgets-8.1.3-py3-none-any.whl", hash = "sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2"}, + {file = "ipywidgets-8.1.3.tar.gz", hash = "sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c"}, +] + +[package.dependencies] +comm = ">=0.1.3" +ipython = ">=6.1.0" +jupyterlab-widgets = ">=3.0.11,<3.1.0" +traitlets = ">=4.3.1" +widgetsnbextension = ">=4.0.11,<4.1.0" + +[package.extras] +test = ["ipykernel", "jsonschema", "pytest (>=3.6.0)", "pytest-cov", "pytz"] + [[package]] name = "isort" version = "5.13.2" @@ -2288,6 +2326,17 @@ traitlets = ">=5.3" docs = ["myst-parser", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] test = ["ipykernel", "pre-commit", "pytest (<8)", "pytest-cov", "pytest-timeout"] +[[package]] +name = "jupyterlab-widgets" +version = "3.0.11" +description = "Jupyter interactive widgets for JupyterLab" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jupyterlab_widgets-3.0.11-py3-none-any.whl", hash = "sha256:78287fd86d20744ace330a61625024cf5521e1c012a352ddc0a3cdc2348becd0"}, + {file = "jupyterlab_widgets-3.0.11.tar.gz", hash = "sha256:dd5ac679593c969af29c9bed054c24f26842baa51352114736756bc035deee27"}, +] + [[package]] name = "keyboard" version = "0.13.5" @@ -3541,22 +3590,32 @@ git-python = ">=1.0.3,<2.0.0" html2image = ">=2.0.4.3,<3.0.0.0" inquirer = ">=3.1.3,<4.0.0" ipykernel = ">=6.26.0,<7.0.0" +ipywidgets = {version = ">=8.1.2,<9.0.0", optional = true, markers = "extra == \"os\""} jupyter-client = ">=8.6.0,<9.0.0" litellm = ">=1.35.32,<2.0.0" matplotlib = ">=3.8.2,<4.0.0" nltk = ">=3.8.1,<4.0.0" +opencv-python = {version = ">=4.8.1.78,<5.0.0.0", optional = true, markers = "extra == \"os\" or extra == \"local\""} platformdirs = ">=4.2.0,<5.0.0" +plyer = {version = ">=2.1.0,<3.0.0", optional = true, markers = "extra == \"os\""} psutil = ">=5.9.6,<6.0.0" +pyautogui = {version = ">=0.9.54,<0.10.0", optional = true, markers = "extra == \"os\""} pydantic = ">=2.6.4,<3.0.0" pyreadline3 = {version = ">=3.4.1,<4.0.0", markers = "sys_platform == \"win32\""} +pytesseract = {version = ">=0.3.10,<0.4.0", optional = true, markers = "extra == \"os\" or extra == \"local\""} +pywinctl = {version = ">=0.3,<0.4", optional = true, markers = "extra == \"os\""} pyyaml = ">=6.0.1,<7.0.0" rich = ">=13.4.2,<14.0.0" +screeninfo = {version = ">=0.8.1,<0.9.0", optional = true, markers = "extra == \"os\""} send2trash = ">=1.8.2,<2.0.0" +sentence-transformers = {version = ">=2.5.1,<3.0.0", optional = true, markers = "extra == \"os\""} setuptools = "*" six = ">=1.16.0,<2.0.0" tiktoken = ">=0.6.0,<0.7.0" +timm = {version = ">=0.9.16,<0.10.0", optional = true, markers = "extra == \"os\""} tokentrim = ">=0.1.13,<0.2.0" toml = ">=0.10.2,<0.11.0" +torch = {version = ">=2.2.1,<3.0.0", optional = true, markers = "extra == \"os\" or extra == \"local\""} wget = ">=3.2,<4.0" [package.extras] @@ -3781,6 +3840,23 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "plyer" +version = "2.1.0" +description = "Platform-independent wrapper for platform-dependent APIs" +optional = false +python-versions = "*" +files = [ + {file = "plyer-2.1.0-py2.py3-none-any.whl", hash = "sha256:1b1772060df8b3045ed4f08231690ec8f7de30f5a004aa1724665a9074eed113"}, + {file = "plyer-2.1.0.tar.gz", hash = "sha256:65b7dfb7e11e07af37a8487eb2aa69524276ef70dad500b07228ce64736baa61"}, +] + +[package.extras] +android = ["pyjnius"] +dev = ["flake8", "mock"] +ios = ["pyobjus"] +macosx = ["pyobjus"] + [[package]] name = "pooch" version = "1.8.1" @@ -4197,6 +4273,26 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pymonctl" +version = "0.92" +description = "Cross-Platform toolkit to get info on and control monitors connected" +optional = false +python-versions = "*" +files = [ + {file = "PyMonCtl-0.92-py3-none-any.whl", hash = "sha256:2495d8dab78f9a7dbce37e74543e60b8bd404a35c3108935697dda7768611b5a"}, +] + +[package.dependencies] +ewmhlib = {version = ">=0.1", markers = "sys_platform == \"linux\""} +pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""} +python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} +pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""} +typing-extensions = ">=4.4.0" + +[package.extras] +dev = ["mypy (>=0.990)", "pywinctl (>=0.3)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"] + [[package]] name = "pymsgbox" version = "1.0.9" @@ -6963,6 +7059,21 @@ Pillow = [ {version = ">=9.3.0", markers = "python_version == \"3.11\""}, ] +[[package]] +name = "pytesseract" +version = "0.3.10" +description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytesseract-0.3.10-py3-none-any.whl", hash = "sha256:8f22cc98f765bf13517ead0c70effedb46c153540d25783e04014f28b55a5fc6"}, + {file = "pytesseract-0.3.10.tar.gz", hash = "sha256:f1c3a8b0f07fd01a1085d451f5b8315be6eec1d5577a6796d46dc7a62bd4120f"}, +] + +[package.dependencies] +packaging = ">=21.3" +Pillow = ">=8.0.0" + [[package]] name = "pytest" version = "8.2.1" @@ -7228,6 +7339,47 @@ files = [ {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, ] +[[package]] +name = "pywinbox" +version = "0.7" +description = "Cross-Platform and multi-monitor toolkit to handle rectangular areas and windows box" +optional = false +python-versions = "*" +files = [ + {file = "PyWinBox-0.7-py3-none-any.whl", hash = "sha256:8b2506a8dd7afa0a910b368762adfac885274132ef9151b0c81b0d2c6ffd6f83"}, +] + +[package.dependencies] +ewmhlib = {version = ">=0.1", markers = "sys_platform == \"linux\""} +pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""} +python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} +pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""} +typing-extensions = ">=4.4.0" + +[package.extras] +dev = ["mypy (>=0.990)", "pywinctl (>=0.3)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"] + +[[package]] +name = "pywinctl" +version = "0.3" +description = "Cross-Platform toolkit to get info on and control windows on screen" +optional = false +python-versions = "*" +files = [ + {file = "PyWinCtl-0.3-py3-none-any.whl", hash = "sha256:3603981c87b0c64987e7be857d89450f98792b01f49006a17dac758e11141dd7"}, +] + +[package.dependencies] +pymonctl = ">=0.6" +pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""} +python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""} +pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""} +pywinbox = ">=0.6" +typing-extensions = ">=4.4.0" + +[package.extras] +dev = ["mypy (>=0.990)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"] + [[package]] name = "pyyaml" version = "6.0.1" @@ -7825,6 +7977,21 @@ dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pyde doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +[[package]] +name = "screeninfo" +version = "0.8.1" +description = "Fetch location and size of physical screens." +optional = false +python-versions = ">=3.6.2,<4.0.0" +files = [ + {file = "screeninfo-0.8.1-py3-none-any.whl", hash = "sha256:e97d6b173856edcfa3bd282f81deb528188aff14b11ec3e195584e7641be733c"}, + {file = "screeninfo-0.8.1.tar.gz", hash = "sha256:9983076bcc7e34402a1a9e4d7dabf3729411fd2abb3f3b4be7eba73519cd2ed1"}, +] + +[package.dependencies] +Cython = {version = "*", markers = "sys_platform == \"darwin\""} +pyobjc-framework-Cocoa = {version = "*", markers = "sys_platform == \"darwin\""} + [[package]] name = "send2trash" version = "1.8.3" @@ -7841,6 +8008,30 @@ nativelib = ["pyobjc-framework-Cocoa", "pywin32"] objc = ["pyobjc-framework-Cocoa"] win32 = ["pywin32"] +[[package]] +name = "sentence-transformers" +version = "2.7.0" +description = "Multilingual text embeddings" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "sentence_transformers-2.7.0-py3-none-any.whl", hash = "sha256:6a7276b05a95931581bbfa4ba49d780b2cf6904fa4a171ec7fd66c343f761c98"}, + {file = "sentence_transformers-2.7.0.tar.gz", hash = "sha256:2f7df99d1c021dded471ed2d079e9d1e4fc8e30ecb06f957be060511b36f24ea"}, +] + +[package.dependencies] +huggingface-hub = ">=0.15.1" +numpy = "*" +Pillow = "*" +scikit-learn = "*" +scipy = "*" +torch = ">=1.11.0" +tqdm = "*" +transformers = ">=4.34.0,<5.0.0" + +[package.extras] +dev = ["pre-commit", "pytest", "ruff (>=0.3.0)"] + [[package]] name = "sentry-sdk" version = "2.4.0" @@ -8561,6 +8752,24 @@ requests = ">=2.26.0" [package.extras] blobfile = ["blobfile (>=2)"] +[[package]] +name = "timm" +version = "0.9.16" +description = "PyTorch Image Models" +optional = false +python-versions = ">=3.8" +files = [ + {file = "timm-0.9.16-py3-none-any.whl", hash = "sha256:bf5704014476ab011589d3c14172ee4c901fd18f9110a928019cac5be2945914"}, + {file = "timm-0.9.16.tar.gz", hash = "sha256:891e54f375d55adf31a71ab0c117761f0e472f9f3971858ecdd1e7376b7071e6"}, +] + +[package.dependencies] +huggingface_hub = "*" +pyyaml = "*" +safetensors = "*" +torch = "*" +torchvision = "*" + [[package]] name = "tokenizers" version = "0.15.2" @@ -8819,6 +9028,48 @@ files = [ [package.dependencies] torch = "2.2.2" +[[package]] +name = "torchvision" +version = "0.17.2" +description = "image and video datasets and models for torch deep learning" +optional = false +python-versions = ">=3.8" +files = [ + {file = "torchvision-0.17.2-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:1f2910fe3c21ad6875b2720d46fad835b2e4b336e9553d31ca364d24c90b1d4f"}, + {file = "torchvision-0.17.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecc1c503fa8a54fbab777e06a7c228032b8ab78efebf35b28bc8f22f544f51f1"}, + {file = "torchvision-0.17.2-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:f400145fc108833e7c2fc28486a04989ca742146d7a2a2cc48878ebbb40cdbbd"}, + {file = "torchvision-0.17.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:e9e4bed404af33dfc92eecc2b513d21ddc4c242a7fd8708b3b09d3a26aa6f444"}, + {file = "torchvision-0.17.2-cp310-cp310-win_amd64.whl", hash = "sha256:ba2e62f233eab3d42b648c122a3a29c47cc108ca314dfd5cbb59cd3a143fd623"}, + {file = "torchvision-0.17.2-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:9b83e55ee7d0a1704f52b9c0ac87388e7a6d1d98a6bde7b0b35f9ab54d7bda54"}, + {file = "torchvision-0.17.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e031004a1bc432c980a7bd642f6c189a3efc316e423fc30b5569837166a4e28d"}, + {file = "torchvision-0.17.2-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:3bbc24b7713e8f22766992562547d8b4b10001208d372fe599255af84bfd1a69"}, + {file = "torchvision-0.17.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:833fd2e4216ced924c8aca0525733fe727f9a1af66dfad7c5be7257e97c39678"}, + {file = "torchvision-0.17.2-cp311-cp311-win_amd64.whl", hash = "sha256:6835897df852fad1015e6a106c167c83848114cbcc7d86112384a973404e4431"}, + {file = "torchvision-0.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14fd1d4a033c325bdba2d03a69c3450cab6d3a625f85cc375781d9237ca5d04d"}, + {file = "torchvision-0.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9c3acbebbe379af112b62b535820174277b1f3eed30df264a4e458d58ee4e5b2"}, + {file = "torchvision-0.17.2-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:77d680adf6ce367166a186d2c7fda3a73807ab9a03b2c31a03fa8812c8c5335b"}, + {file = "torchvision-0.17.2-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:f1c9ab3152cfb27f83aca072cac93a3a4c4e4ab0261cf0f2d516b9868a4e96f3"}, + {file = "torchvision-0.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:3f784381419f3ed3f2ec2aa42fb4aeec5bf4135e298d1631e41c926e6f1a0dff"}, + {file = "torchvision-0.17.2-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:b83aac8d78f48981146d582168d75b6c947cfb0a7693f76e219f1926f6e595a3"}, + {file = "torchvision-0.17.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1ece40557e122d79975860a005aa7e2a9e2e6c350a03e78a00ec1450083312fd"}, + {file = "torchvision-0.17.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:32dbeba3987e20f2dc1bce8d1504139fff582898346dfe8ad98d649f97ca78fa"}, + {file = "torchvision-0.17.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:35ba5c1600c3203549d2316422a659bd20c0cfda1b6085eec94fb9f35f55ca43"}, + {file = "torchvision-0.17.2-cp38-cp38-win_amd64.whl", hash = "sha256:2f69570f50b1d195e51bc03feffb7b7728207bc36efcfb1f0813712b2379d881"}, + {file = "torchvision-0.17.2-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:4868bbfa55758c8107e69a0e7dd5e77b89056035cd38b767ad5b98cdb71c0f0d"}, + {file = "torchvision-0.17.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:efd6d0dd0668e15d01a2cffadc74068433b32cbcf5692e0c4aa15fc5cb250ce7"}, + {file = "torchvision-0.17.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7dc85b397f6c6d9ef12716ce0d6e11ac2b803f5cccff6fe3966db248e7774478"}, + {file = "torchvision-0.17.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d506854c5acd69b20a8b6641f01fe841685a21c5406b56813184f1c9fc94279e"}, + {file = "torchvision-0.17.2-cp39-cp39-win_amd64.whl", hash = "sha256:067095e87a020a7a251ac1d38483aa591c5ccb81e815527c54db88a982fc9267"}, +] + +[package.dependencies] +numpy = "*" +pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" +torch = "2.2.2" + +[package.extras] +scipy = ["scipy"] + [[package]] name = "tornado" version = "6.4" @@ -9322,6 +9573,17 @@ files = [ {file = "wget-3.2.zip", hash = "sha256:35e630eca2aa50ce998b9b1a127bb26b30dfee573702782aa982f875e3f16061"}, ] +[[package]] +name = "widgetsnbextension" +version = "4.0.11" +description = "Jupyter interactive widgets for Jupyter Notebook" +optional = false +python-versions = ">=3.7" +files = [ + {file = "widgetsnbextension-4.0.11-py3-none-any.whl", hash = "sha256:55d4d6949d100e0d08b94948a42efc3ed6dfdc0e9468b2c4b128c9a2ce3a7a36"}, + {file = "widgetsnbextension-4.0.11.tar.gz", hash = "sha256:8b22a8f1910bfd188e596fe7fc05dcbd87e810c8a4ba010bdb3da86637398474"}, +] + [[package]] name = "xmod" version = "1.8.1" @@ -9454,4 +9716,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "de07d24f74d39335d2c89d83cbe681bacbeaeffdb30227cb9f6955dc4614dfca" +content-hash = "e6b8e346d63f867e76a757bb39c2db877d51b07b549642a409685ed4aa1d0bbe" diff --git a/software/pyproject.toml b/software/pyproject.toml index ab54d5c..379769f 100644 --- a/software/pyproject.toml +++ b/software/pyproject.toml @@ -40,13 +40,16 @@ ctranslate2 = "4.1.0" py3-tts = "^3.5" elevenlabs = "1.2.2" groq = "^0.5.0" -open-interpreter = "^0.2.6" +open-interpreter = {extras = ["os"], version = "^0.2.6"} litellm = "1.35.35" openai = "1.30.5" pywebview = "*" pyobjc = "*" sentry-sdk = "^2.4.0" +plyer = "^2.1.0" +pywinctl = "^0.3" +dateparser = "^1.1.0" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/software/source/clients/base_device.py b/software/source/clients/base_device.py index 885753e..691f501 100644 --- a/software/source/clients/base_device.py +++ b/software/source/clients/base_device.py @@ -2,6 +2,7 @@ from dotenv import load_dotenv load_dotenv() # take environment variables from .env. +import requests import subprocess import os import sys @@ -12,6 +13,7 @@ from pynput import keyboard import json import traceback import websockets +import websockets.sync.client import queue from pydub import AudioSegment from pydub.playback import play @@ -91,7 +93,7 @@ class Device: self.server_url = "" self.ctrl_pressed = False self.tts_service = "" - self.playback_latency = None + # self.playback_latency = None def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX): """Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list.""" @@ -165,6 +167,7 @@ class Device: while True: try: audio = await self.audiosegments.get() + if self.playback_latency and isinstance(audio, bytes): elapsed_time = time.time() - self.playback_latency print(f"Time from request to playback: {elapsed_time} seconds") @@ -224,7 +227,7 @@ class Device: stream.stop_stream() stream.close() print("Recording stopped.") - self.playback_latency = time.time() + # self.playback_latency = time.time() duration = wav_file.getnframes() / RATE if duration < 0.3: @@ -412,7 +415,7 @@ class Device: # See https://github.com/OpenInterpreter/01/issues/197 try: ws = websockets.connect(WS_URL) - await exec_ws_communication(ws) + # await exec_ws_communication(ws) except Exception as e: logger.error(f"Error while attempting to connect: {e}") else: @@ -431,7 +434,7 @@ class Device: # Configuration for WebSocket WS_URL = f"ws://{self.server_url}" # Start the WebSocket communication - asyncio.create_task(self.websocket_communication(WS_URL)) + await self.websocket_communication(WS_URL) # Start watching the kernel if it's your job to do that if os.getenv("CODE_RUNNER") == "client": diff --git a/software/source/server/async_interpreter.py b/software/source/server/async_interpreter.py index 39303d2..71cb0ed 100644 --- a/software/source/server/async_interpreter.py +++ b/software/source/server/async_interpreter.py @@ -32,7 +32,7 @@ class AsyncInterpreter: self.stt = AudioToTextRecorder( model="tiny.en", spinner=False, use_microphone=False ) - self.stt.stop() # It needs this for some reason + self.stt.stop() # TTS if self.interpreter.tts == "coqui": diff --git a/software/source/server/async_server.py b/software/source/server/async_server.py index 13fcecd..0228489 100644 --- a/software/source/server/async_server.py +++ b/software/source/server/async_server.py @@ -13,7 +13,9 @@ from fastapi import FastAPI, WebSocket from fastapi.responses import PlainTextResponse from uvicorn import Config, Server + # from interpreter import interpreter as base_interpreter + from .async_interpreter import AsyncInterpreter from fastapi.middleware.cors import CORSMiddleware from typing import List, Dict, Any @@ -49,47 +51,105 @@ async def main(server_host, server_port, tts_service): print("🪼🪼🪼🪼🪼🪼 Messages loaded: ", interpreter.active_chat_messages) return {"status": "success"} + print("About to set up the websocker endpoint!!!!!!!!!!!!!!!!!!!!!!!!!") + @app.websocket("/") async def websocket_endpoint(websocket: WebSocket): + print("websocket hit") await websocket.accept() - try: + print("websocket accepted") - async def receive_input(): + async def send_output(): + try: while True: - if websocket.client_state == "DISCONNECTED": - break + output = await interpreter.output() + if isinstance(output, bytes): + print("server sending bytes output") + try: + await websocket.send_bytes(output) + print("server successfully sent bytes output") + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + + elif isinstance(output, dict): + print("server sending text output") + try: + await websocket.send_text(json.dumps(output)) + print("server successfully sent text output") + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + except asyncio.CancelledError: + print("WebSocket connection closed") + traceback.print_exc() + + async def receive_input(): + try: + while True: + # print("server awaiting input") data = await websocket.receive() if isinstance(data, bytes): - await interpreter.input(data) + try: + await interpreter.input(data) + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + elif "bytes" in data: - await interpreter.input(data["bytes"]) - # print("RECEIVED INPUT", data) + try: + await interpreter.input(data["bytes"]) + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + elif "text" in data: - # print("RECEIVED INPUT", data) - await interpreter.input(data["text"]) + try: + await interpreter.input(data["text"]) + except Exception as e: + print(f"Error: {e}") + traceback.print_exc() + return {"error": str(e)} + except asyncio.CancelledError: + print("WebSocket connection closed") + traceback.print_exc() - async def send_output(): - while True: - output = await interpreter.output() + try: + send_task = asyncio.create_task(send_output()) + receive_task = asyncio.create_task(receive_input()) - if isinstance(output, bytes): - # print(f"Sending {len(output)} bytes of audio data.") - await websocket.send_bytes(output) - # we dont send out bytes rn, no TTS + print("server starting to handle ws connection") + """ + done, pending = await asyncio.wait( + [send_task, receive_task], + return_when=asyncio.FIRST_COMPLETED, + ) - elif isinstance(output, dict): - # print("sending text") - await websocket.send_text(json.dumps(output)) + for task in pending: + task.cancel() + + for task in done: + if task.exception() is not None: + raise + """ + await asyncio.gather(send_task, receive_task) + + print("server finished handling ws connection") - await asyncio.gather(send_output(), receive_input()) + except WebSocketDisconnect: + print("WebSocket disconnected") except Exception as e: print(f"WebSocket connection closed with exception: {e}") traceback.print_exc() finally: - if not websocket.client_state == "DISCONNECTED": - await websocket.close() + print("server closing ws connection") + await websocket.close() print(f"Starting server on {server_host}:{server_port}") config = Config(app, host=server_host, port=server_port, lifespan="on") @@ -98,4 +158,4 @@ async def main(server_host, server_port, tts_service): if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main("localhost", 8000))