Merge branch 'async-interpreter' into temp-branch

software/poetry.lock

@@ -1286,6 +1286,23 @@ files = [
     {file = "evdev-1.7.1.tar.gz", hash = "sha256:0c72c370bda29d857e188d931019c32651a9c1ea977c08c8d939b1ced1637fde"},
 ]
 
+[[package]]
+name = "ewmhlib"
+version = "0.2"
+description = "Extended Window Manager Hints implementation in Python 3"
+optional = false
+python-versions = "*"
+files = [
+    {file = "EWMHlib-0.2-py3-none-any.whl", hash = "sha256:f5b07d8cfd4c7734462ee744c32d490f2f3233fa7ab354240069344208d2f6f5"},
+]
+
+[package.dependencies]
+python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""}
+typing-extensions = ">=4.4.0"
+
+[package.extras]
+dev = ["mypy (>=0.990)", "types-python-xlib (>=0.32)", "types-setuptools (>=65.5)"]
+
 [[package]]
 name = "exceptiongroup"
 version = "1.2.1"
@@ -2156,6 +2173,27 @@ qtconsole = ["qtconsole"]
 test = ["pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath"]
 test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7.1)", "pytest-asyncio (<0.22)", "testpath", "trio"]
 
+[[package]]
+name = "ipywidgets"
+version = "8.1.3"
+description = "Jupyter interactive widgets"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "ipywidgets-8.1.3-py3-none-any.whl", hash = "sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2"},
+    {file = "ipywidgets-8.1.3.tar.gz", hash = "sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c"},
+]
+
+[package.dependencies]
+comm = ">=0.1.3"
+ipython = ">=6.1.0"
+jupyterlab-widgets = ">=3.0.11,<3.1.0"
+traitlets = ">=4.3.1"
+widgetsnbextension = ">=4.0.11,<4.1.0"
+
+[package.extras]
+test = ["ipykernel", "jsonschema", "pytest (>=3.6.0)", "pytest-cov", "pytz"]
+
 [[package]]
 name = "isort"
 version = "5.13.2"
@@ -2288,6 +2326,17 @@ traitlets = ">=5.3"
 docs = ["myst-parser", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"]
 test = ["ipykernel", "pre-commit", "pytest (<8)", "pytest-cov", "pytest-timeout"]
 
+[[package]]
+name = "jupyterlab-widgets"
+version = "3.0.11"
+description = "Jupyter interactive widgets for JupyterLab"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "jupyterlab_widgets-3.0.11-py3-none-any.whl", hash = "sha256:78287fd86d20744ace330a61625024cf5521e1c012a352ddc0a3cdc2348becd0"},
+    {file = "jupyterlab_widgets-3.0.11.tar.gz", hash = "sha256:dd5ac679593c969af29c9bed054c24f26842baa51352114736756bc035deee27"},
+]
+
 [[package]]
 name = "keyboard"
 version = "0.13.5"
@@ -3541,22 +3590,32 @@ git-python = ">=1.0.3,<2.0.0"
 html2image = ">=2.0.4.3,<3.0.0.0"
 inquirer = ">=3.1.3,<4.0.0"
 ipykernel = ">=6.26.0,<7.0.0"
+ipywidgets = {version = ">=8.1.2,<9.0.0", optional = true, markers = "extra == \"os\""}
 jupyter-client = ">=8.6.0,<9.0.0"
 litellm = ">=1.35.32,<2.0.0"
 matplotlib = ">=3.8.2,<4.0.0"
 nltk = ">=3.8.1,<4.0.0"
+opencv-python = {version = ">=4.8.1.78,<5.0.0.0", optional = true, markers = "extra == \"os\" or extra == \"local\""}
 platformdirs = ">=4.2.0,<5.0.0"
+plyer = {version = ">=2.1.0,<3.0.0", optional = true, markers = "extra == \"os\""}
 psutil = ">=5.9.6,<6.0.0"
+pyautogui = {version = ">=0.9.54,<0.10.0", optional = true, markers = "extra == \"os\""}
 pydantic = ">=2.6.4,<3.0.0"
 pyreadline3 = {version = ">=3.4.1,<4.0.0", markers = "sys_platform == \"win32\""}
+pytesseract = {version = ">=0.3.10,<0.4.0", optional = true, markers = "extra == \"os\" or extra == \"local\""}
+pywinctl = {version = ">=0.3,<0.4", optional = true, markers = "extra == \"os\""}
 pyyaml = ">=6.0.1,<7.0.0"
 rich = ">=13.4.2,<14.0.0"
+screeninfo = {version = ">=0.8.1,<0.9.0", optional = true, markers = "extra == \"os\""}
 send2trash = ">=1.8.2,<2.0.0"
+sentence-transformers = {version = ">=2.5.1,<3.0.0", optional = true, markers = "extra == \"os\""}
 setuptools = "*"
 six = ">=1.16.0,<2.0.0"
 tiktoken = ">=0.6.0,<0.7.0"
+timm = {version = ">=0.9.16,<0.10.0", optional = true, markers = "extra == \"os\""}
 tokentrim = ">=0.1.13,<0.2.0"
 toml = ">=0.10.2,<0.11.0"
+torch = {version = ">=2.2.1,<3.0.0", optional = true, markers = "extra == \"os\" or extra == \"local\""}
 wget = ">=3.2,<4.0"
 
 [package.extras]
@@ -3781,6 +3840,23 @@ files = [
 dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
+[[package]]
+name = "plyer"
+version = "2.1.0"
+description = "Platform-independent wrapper for platform-dependent APIs"
+optional = false
+python-versions = "*"
+files = [
+    {file = "plyer-2.1.0-py2.py3-none-any.whl", hash = "sha256:1b1772060df8b3045ed4f08231690ec8f7de30f5a004aa1724665a9074eed113"},
+    {file = "plyer-2.1.0.tar.gz", hash = "sha256:65b7dfb7e11e07af37a8487eb2aa69524276ef70dad500b07228ce64736baa61"},
+]
+
+[package.extras]
+android = ["pyjnius"]
+dev = ["flake8", "mock"]
+ios = ["pyobjus"]
+macosx = ["pyobjus"]
+
 [[package]]
 name = "pooch"
 version = "1.8.1"
@@ -4197,6 +4273,26 @@ files = [
 [package.extras]
 windows-terminal = ["colorama (>=0.4.6)"]
 
+[[package]]
+name = "pymonctl"
+version = "0.92"
+description = "Cross-Platform toolkit to get info on and control monitors connected"
+optional = false
+python-versions = "*"
+files = [
+    {file = "PyMonCtl-0.92-py3-none-any.whl", hash = "sha256:2495d8dab78f9a7dbce37e74543e60b8bd404a35c3108935697dda7768611b5a"},
+]
+
+[package.dependencies]
+ewmhlib = {version = ">=0.1", markers = "sys_platform == \"linux\""}
+pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""}
+python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""}
+pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""}
+typing-extensions = ">=4.4.0"
+
+[package.extras]
+dev = ["mypy (>=0.990)", "pywinctl (>=0.3)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"]
+
 [[package]]
 name = "pymsgbox"
 version = "1.0.9"
@@ -6963,6 +7059,21 @@ Pillow = [
     {version = ">=9.3.0", markers = "python_version == \"3.11\""},
 ]
 
+[[package]]
+name = "pytesseract"
+version = "0.3.10"
+description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytesseract-0.3.10-py3-none-any.whl", hash = "sha256:8f22cc98f765bf13517ead0c70effedb46c153540d25783e04014f28b55a5fc6"},
+    {file = "pytesseract-0.3.10.tar.gz", hash = "sha256:f1c3a8b0f07fd01a1085d451f5b8315be6eec1d5577a6796d46dc7a62bd4120f"},
+]
+
+[package.dependencies]
+packaging = ">=21.3"
+Pillow = ">=8.0.0"
+
 [[package]]
 name = "pytest"
 version = "8.2.1"
@@ -7228,6 +7339,47 @@ files = [
     {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"},
 ]
 
+[[package]]
+name = "pywinbox"
+version = "0.7"
+description = "Cross-Platform and multi-monitor toolkit to handle rectangular areas and windows box"
+optional = false
+python-versions = "*"
+files = [
+    {file = "PyWinBox-0.7-py3-none-any.whl", hash = "sha256:8b2506a8dd7afa0a910b368762adfac885274132ef9151b0c81b0d2c6ffd6f83"},
+]
+
+[package.dependencies]
+ewmhlib = {version = ">=0.1", markers = "sys_platform == \"linux\""}
+pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""}
+python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""}
+pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""}
+typing-extensions = ">=4.4.0"
+
+[package.extras]
+dev = ["mypy (>=0.990)", "pywinctl (>=0.3)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"]
+
+[[package]]
+name = "pywinctl"
+version = "0.3"
+description = "Cross-Platform toolkit to get info on and control windows on screen"
+optional = false
+python-versions = "*"
+files = [
+    {file = "PyWinCtl-0.3-py3-none-any.whl", hash = "sha256:3603981c87b0c64987e7be857d89450f98792b01f49006a17dac758e11141dd7"},
+]
+
+[package.dependencies]
+pymonctl = ">=0.6"
+pyobjc = {version = ">=8.1", markers = "sys_platform == \"darwin\""}
+python-xlib = {version = ">=0.21", markers = "sys_platform == \"linux\""}
+pywin32 = {version = ">=302", markers = "sys_platform == \"win32\""}
+pywinbox = ">=0.6"
+typing-extensions = ">=4.4.0"
+
+[package.extras]
+dev = ["mypy (>=0.990)", "types-python-xlib (>=0.32)", "types-pywin32 (>=305.0.0.3)", "types-setuptools (>=65.5)"]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.1"
@@ -7825,6 +7977,21 @@ dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pyde
 doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"]
 test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"]
 
+[[package]]
+name = "screeninfo"
+version = "0.8.1"
+description = "Fetch location and size of physical screens."
+optional = false
+python-versions = ">=3.6.2,<4.0.0"
+files = [
+    {file = "screeninfo-0.8.1-py3-none-any.whl", hash = "sha256:e97d6b173856edcfa3bd282f81deb528188aff14b11ec3e195584e7641be733c"},
+    {file = "screeninfo-0.8.1.tar.gz", hash = "sha256:9983076bcc7e34402a1a9e4d7dabf3729411fd2abb3f3b4be7eba73519cd2ed1"},
+]
+
+[package.dependencies]
+Cython = {version = "*", markers = "sys_platform == \"darwin\""}
+pyobjc-framework-Cocoa = {version = "*", markers = "sys_platform == \"darwin\""}
+
 [[package]]
 name = "send2trash"
 version = "1.8.3"
@@ -7841,6 +8008,30 @@ nativelib = ["pyobjc-framework-Cocoa", "pywin32"]
 objc = ["pyobjc-framework-Cocoa"]
 win32 = ["pywin32"]
 
+[[package]]
+name = "sentence-transformers"
+version = "2.7.0"
+description = "Multilingual text embeddings"
+optional = false
+python-versions = ">=3.8.0"
+files = [
+    {file = "sentence_transformers-2.7.0-py3-none-any.whl", hash = "sha256:6a7276b05a95931581bbfa4ba49d780b2cf6904fa4a171ec7fd66c343f761c98"},
+    {file = "sentence_transformers-2.7.0.tar.gz", hash = "sha256:2f7df99d1c021dded471ed2d079e9d1e4fc8e30ecb06f957be060511b36f24ea"},
+]
+
+[package.dependencies]
+huggingface-hub = ">=0.15.1"
+numpy = "*"
+Pillow = "*"
+scikit-learn = "*"
+scipy = "*"
+torch = ">=1.11.0"
+tqdm = "*"
+transformers = ">=4.34.0,<5.0.0"
+
+[package.extras]
+dev = ["pre-commit", "pytest", "ruff (>=0.3.0)"]
+
 [[package]]
 name = "sentry-sdk"
 version = "2.4.0"
@@ -8561,6 +8752,24 @@ requests = ">=2.26.0"
 [package.extras]
 blobfile = ["blobfile (>=2)"]
 
+[[package]]
+name = "timm"
+version = "0.9.16"
+description = "PyTorch Image Models"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "timm-0.9.16-py3-none-any.whl", hash = "sha256:bf5704014476ab011589d3c14172ee4c901fd18f9110a928019cac5be2945914"},
+    {file = "timm-0.9.16.tar.gz", hash = "sha256:891e54f375d55adf31a71ab0c117761f0e472f9f3971858ecdd1e7376b7071e6"},
+]
+
+[package.dependencies]
+huggingface_hub = "*"
+pyyaml = "*"
+safetensors = "*"
+torch = "*"
+torchvision = "*"
+
 [[package]]
 name = "tokenizers"
 version = "0.15.2"
@@ -8819,6 +9028,48 @@ files = [
 [package.dependencies]
 torch = "2.2.2"
 
+[[package]]
+name = "torchvision"
+version = "0.17.2"
+description = "image and video datasets and models for torch deep learning"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "torchvision-0.17.2-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:1f2910fe3c21ad6875b2720d46fad835b2e4b336e9553d31ca364d24c90b1d4f"},
+    {file = "torchvision-0.17.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecc1c503fa8a54fbab777e06a7c228032b8ab78efebf35b28bc8f22f544f51f1"},
+    {file = "torchvision-0.17.2-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:f400145fc108833e7c2fc28486a04989ca742146d7a2a2cc48878ebbb40cdbbd"},
+    {file = "torchvision-0.17.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:e9e4bed404af33dfc92eecc2b513d21ddc4c242a7fd8708b3b09d3a26aa6f444"},
+    {file = "torchvision-0.17.2-cp310-cp310-win_amd64.whl", hash = "sha256:ba2e62f233eab3d42b648c122a3a29c47cc108ca314dfd5cbb59cd3a143fd623"},
+    {file = "torchvision-0.17.2-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:9b83e55ee7d0a1704f52b9c0ac87388e7a6d1d98a6bde7b0b35f9ab54d7bda54"},
+    {file = "torchvision-0.17.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e031004a1bc432c980a7bd642f6c189a3efc316e423fc30b5569837166a4e28d"},
+    {file = "torchvision-0.17.2-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:3bbc24b7713e8f22766992562547d8b4b10001208d372fe599255af84bfd1a69"},
+    {file = "torchvision-0.17.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:833fd2e4216ced924c8aca0525733fe727f9a1af66dfad7c5be7257e97c39678"},
+    {file = "torchvision-0.17.2-cp311-cp311-win_amd64.whl", hash = "sha256:6835897df852fad1015e6a106c167c83848114cbcc7d86112384a973404e4431"},
+    {file = "torchvision-0.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:14fd1d4a033c325bdba2d03a69c3450cab6d3a625f85cc375781d9237ca5d04d"},
+    {file = "torchvision-0.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9c3acbebbe379af112b62b535820174277b1f3eed30df264a4e458d58ee4e5b2"},
+    {file = "torchvision-0.17.2-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:77d680adf6ce367166a186d2c7fda3a73807ab9a03b2c31a03fa8812c8c5335b"},
+    {file = "torchvision-0.17.2-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:f1c9ab3152cfb27f83aca072cac93a3a4c4e4ab0261cf0f2d516b9868a4e96f3"},
+    {file = "torchvision-0.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:3f784381419f3ed3f2ec2aa42fb4aeec5bf4135e298d1631e41c926e6f1a0dff"},
+    {file = "torchvision-0.17.2-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:b83aac8d78f48981146d582168d75b6c947cfb0a7693f76e219f1926f6e595a3"},
+    {file = "torchvision-0.17.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1ece40557e122d79975860a005aa7e2a9e2e6c350a03e78a00ec1450083312fd"},
+    {file = "torchvision-0.17.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:32dbeba3987e20f2dc1bce8d1504139fff582898346dfe8ad98d649f97ca78fa"},
+    {file = "torchvision-0.17.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:35ba5c1600c3203549d2316422a659bd20c0cfda1b6085eec94fb9f35f55ca43"},
+    {file = "torchvision-0.17.2-cp38-cp38-win_amd64.whl", hash = "sha256:2f69570f50b1d195e51bc03feffb7b7728207bc36efcfb1f0813712b2379d881"},
+    {file = "torchvision-0.17.2-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:4868bbfa55758c8107e69a0e7dd5e77b89056035cd38b767ad5b98cdb71c0f0d"},
+    {file = "torchvision-0.17.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:efd6d0dd0668e15d01a2cffadc74068433b32cbcf5692e0c4aa15fc5cb250ce7"},
+    {file = "torchvision-0.17.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7dc85b397f6c6d9ef12716ce0d6e11ac2b803f5cccff6fe3966db248e7774478"},
+    {file = "torchvision-0.17.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d506854c5acd69b20a8b6641f01fe841685a21c5406b56813184f1c9fc94279e"},
+    {file = "torchvision-0.17.2-cp39-cp39-win_amd64.whl", hash = "sha256:067095e87a020a7a251ac1d38483aa591c5ccb81e815527c54db88a982fc9267"},
+]
+
+[package.dependencies]
+numpy = "*"
+pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0"
+torch = "2.2.2"
+
+[package.extras]
+scipy = ["scipy"]
+
 [[package]]
 name = "tornado"
 version = "6.4"
@@ -9322,6 +9573,17 @@ files = [
     {file = "wget-3.2.zip", hash = "sha256:35e630eca2aa50ce998b9b1a127bb26b30dfee573702782aa982f875e3f16061"},
 ]
 
+[[package]]
+name = "widgetsnbextension"
+version = "4.0.11"
+description = "Jupyter interactive widgets for Jupyter Notebook"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "widgetsnbextension-4.0.11-py3-none-any.whl", hash = "sha256:55d4d6949d100e0d08b94948a42efc3ed6dfdc0e9468b2c4b128c9a2ce3a7a36"},
+    {file = "widgetsnbextension-4.0.11.tar.gz", hash = "sha256:8b22a8f1910bfd188e596fe7fc05dcbd87e810c8a4ba010bdb3da86637398474"},
+]
+
 [[package]]
 name = "xmod"
 version = "1.8.1"
@@ -9454,4 +9716,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.12"
-content-hash = "de07d24f74d39335d2c89d83cbe681bacbeaeffdb30227cb9f6955dc4614dfca"
+content-hash = "e6b8e346d63f867e76a757bb39c2db877d51b07b549642a409685ed4aa1d0bbe"

software/pyproject.toml

@@ -40,13 +40,16 @@ ctranslate2 = "4.1.0"
 py3-tts = "^3.5"
 elevenlabs = "1.2.2"
 groq = "^0.5.0"
-open-interpreter = "^0.2.6"
+open-interpreter = {extras = ["os"], version = "^0.2.6"}
 litellm = "1.35.35"
 openai = "1.30.5"
 pywebview = "*"
 pyobjc = "*"
 
 sentry-sdk = "^2.4.0"
+plyer = "^2.1.0"
+pywinctl = "^0.3"
+dateparser = "^1.1.0"
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

software/source/clients/base_device.py

@@ -2,6 +2,7 @@ from dotenv import load_dotenv
 
 load_dotenv()  # take environment variables from .env.
 
+import requests
 import subprocess
 import os
 import sys
@@ -12,6 +13,7 @@ from pynput import keyboard
 import json
 import traceback
 import websockets
+import websockets.sync.client
 import queue
 from pydub import AudioSegment
 from pydub.playback import play
@@ -91,7 +93,7 @@ class Device:
         self.server_url = ""
         self.ctrl_pressed = False
         self.tts_service = ""
-        self.playback_latency = None
+        # self.playback_latency = None
 
     def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX):
         """Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list."""
@@ -165,6 +167,7 @@ class Device:
         while True:
             try:
                 audio = await self.audiosegments.get()
+
                 if self.playback_latency and isinstance(audio, bytes):
                     elapsed_time = time.time() - self.playback_latency
                     print(f"Time from request to playback: {elapsed_time} seconds")
@@ -224,7 +227,7 @@ class Device:
         stream.stop_stream()
         stream.close()
         print("Recording stopped.")
-        self.playback_latency = time.time()
+        # self.playback_latency = time.time()
 
         duration = wav_file.getnframes() / RATE
         if duration < 0.3:
@@ -412,7 +415,7 @@ class Device:
             # See https://github.com/OpenInterpreter/01/issues/197
             try:
                 ws = websockets.connect(WS_URL)
-                await exec_ws_communication(ws)
+                # await exec_ws_communication(ws)
             except Exception as e:
                 logger.error(f"Error while attempting to connect: {e}")
         else:
@@ -431,7 +434,7 @@ class Device:
         # Configuration for WebSocket
         WS_URL = f"ws://{self.server_url}"
         # Start the WebSocket communication
-        asyncio.create_task(self.websocket_communication(WS_URL))
+        await self.websocket_communication(WS_URL)
 
         # Start watching the kernel if it's your job to do that
         if os.getenv("CODE_RUNNER") == "client":

software/source/server/async_interpreter.py

@@ -32,7 +32,7 @@ class AsyncInterpreter:
         self.stt = AudioToTextRecorder(
             model="tiny.en", spinner=False, use_microphone=False
         )
-        self.stt.stop()  # It needs this for some reason
+        self.stt.stop()
 
         # TTS
         if self.interpreter.tts == "coqui":

software/source/server/async_server.py

@@ -13,7 +13,9 @@ from fastapi import FastAPI, WebSocket
 from fastapi.responses import PlainTextResponse
 from uvicorn import Config, Server
 
+
 # from interpreter import interpreter as base_interpreter
+
 from .async_interpreter import AsyncInterpreter
 from fastapi.middleware.cors import CORSMiddleware
 from typing import List, Dict, Any
@@ -49,47 +51,105 @@ async def main(server_host, server_port, tts_service):
         print("🪼🪼🪼🪼🪼🪼 Messages loaded: ", interpreter.active_chat_messages)
         return {"status": "success"}
 
+    print("About to set up the websocker endpoint!!!!!!!!!!!!!!!!!!!!!!!!!")
+
     @app.websocket("/")
     async def websocket_endpoint(websocket: WebSocket):
+        print("websocket hit")
         await websocket.accept()
-        try:
+        print("websocket accepted")
 
-            async def receive_input():
+        async def send_output():
+            try:
                 while True:
-                    if websocket.client_state == "DISCONNECTED":
+                    output = await interpreter.output()
-                        break
 
+                    if isinstance(output, bytes):
+                        print("server sending bytes output")
+                        try:
+                            await websocket.send_bytes(output)
+                            print("server successfully sent bytes output")
+                        except Exception as e:
+                            print(f"Error: {e}")
+                            traceback.print_exc()
+                            return {"error": str(e)}
+
+                    elif isinstance(output, dict):
+                        print("server sending text output")
+                        try:
+                            await websocket.send_text(json.dumps(output))
+                            print("server successfully sent text output")
+                        except Exception as e:
+                            print(f"Error: {e}")
+                            traceback.print_exc()
+                            return {"error": str(e)}
+            except asyncio.CancelledError:
+                print("WebSocket connection closed")
+                traceback.print_exc()
+
+        async def receive_input():
+            try:
+                while True:
+                    # print("server awaiting input")
                     data = await websocket.receive()
 
                     if isinstance(data, bytes):
-                        await interpreter.input(data)
+                        try:
+                            await interpreter.input(data)
+                        except Exception as e:
+                            print(f"Error: {e}")
+                            traceback.print_exc()
+                            return {"error": str(e)}
+
                     elif "bytes" in data:
-                        await interpreter.input(data["bytes"])
+                        try:
-                        # print("RECEIVED INPUT", data)
+                            await interpreter.input(data["bytes"])
+                        except Exception as e:
+                            print(f"Error: {e}")
+                            traceback.print_exc()
+                            return {"error": str(e)}
+
                     elif "text" in data:
-                        # print("RECEIVED INPUT", data)
+                        try:
-                        await interpreter.input(data["text"])
+                            await interpreter.input(data["text"])
+                        except Exception as e:
+                            print(f"Error: {e}")
+                            traceback.print_exc()
+                            return {"error": str(e)}
+            except asyncio.CancelledError:
+                print("WebSocket connection closed")
+                traceback.print_exc()
 
-            async def send_output():
+        try:
-                while True:
+            send_task = asyncio.create_task(send_output())
-                    output = await interpreter.output()
+            receive_task = asyncio.create_task(receive_input())
 
-                    if isinstance(output, bytes):
+            print("server starting to handle ws connection")
-                        # print(f"Sending {len(output)} bytes of audio data.")
+            """
-                        await websocket.send_bytes(output)
+            done, pending = await asyncio.wait(
-                        # we dont send out bytes rn, no TTS
+                [send_task, receive_task],
+                return_when=asyncio.FIRST_COMPLETED,
+            )
 
-                    elif isinstance(output, dict):
+            for task in pending:
-                        # print("sending text")
+                task.cancel()
-                        await websocket.send_text(json.dumps(output))
+
+            for task in done:
+                if task.exception() is not None:
+                    raise
+            """
+            await asyncio.gather(send_task, receive_task)
+
+            print("server finished handling ws connection")
 
-            await asyncio.gather(send_output(), receive_input())
+        except WebSocketDisconnect:
+            print("WebSocket disconnected")
         except Exception as e:
             print(f"WebSocket connection closed with exception: {e}")
             traceback.print_exc()
         finally:
-            if not websocket.client_state == "DISCONNECTED":
+            print("server closing ws connection")
-                await websocket.close()
+            await websocket.close()
 
     print(f"Starting server on {server_host}:{server_port}")
     config = Config(app, host=server_host, port=server_port, lifespan="on")
@@ -98,4 +158,4 @@ async def main(server_host, server_port, tts_service):
 
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(main("localhost", 8000))