From e29f82ee77c06e148ddd408bc0c4db1571ad26c8 Mon Sep 17 00:00:00 2001
From: Maki
Date: Wed, 20 Mar 2024 01:14:46 +0900
Subject: [PATCH 01/39] Create README_JP.md
---
docs/README_JP.md | 192 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 192 insertions(+)
create mode 100644 docs/README_JP.md
diff --git a/docs/README_JP.md b/docs/README_JP.md
new file mode 100644
index 0000000..bd7c689
--- /dev/null
+++ b/docs/README_JP.md
@@ -0,0 +1,192 @@
+はい、以下が自然な日本語に変換した文章になります。
+
+* * *
+
+[The 01 Project](https://twitter.com/hellokillian/status/1745875973583896950)の公式プレリリースリポジトリ。
+
+> 発売まで残り**3**日
+
+○
+
+
+
+
+
+ オープンソースの言語モデルコンピューター
+
+
+
+
+
+ | [日本語](README_JP.md) | [English](../README.md) |
+
+
+
+
+
+![ポスター](https://pbs.twimg.com/media/GDqTVYzbgAIfLJf?format=png&name=4096x4096)
+
+
+
+
+
+```shell
+git clone https://github.com/OpenInterpreter/01
+cd 01/01OS
+```
+
+
+
+```shell
+poetry install
+poetry run 01
+```
+
+
+
+**The 01 Project**は、AIデバイス向けのエコシステムを構築しています。
+
+私たちの主力オペレーティングシステムは、Rabbit R1、Humane Pin、[Star Trekコンピューター](https://www.youtube.com/watch?v=1ZXugicgn6U)のような会話型デバイスを動作させることができます。
+
+私たちは、オープンソース、モジュール性、無料であり続けることを約束することで、この分野のGNU/Linuxになることを目指しています。
+
+## 統一API
+
+統一APIは、01で使用される主要サービスの標準的なPythonインターフェースです。
+
+- `/stt` 音声認識用
+- `/llm` 言語モデル用
+- `/tts` 音声合成用
+
+## ボディ
+
+01OSは、さまざまなボディに収容できます。このリストに追加するPRを大歓迎します。
+
+**01 Light**は、ESP32ベースの音声インターフェースで、インターネット経由でホームコンピューターを制御します。**01 Server**と組み合わせて使用します。
+
+**01 Heavy**は、すべてをローカルで実行するデバイスです。
+
+## セットアップ
+
+### 依存関係のインストール
+
+```bash
+# MacOS
+brew install portaudio ffmpeg cmake
+
+# Ubuntu
+sudo apt-get install portaudio19-dev ffmpeg cmake
+```
+
+Whisperを使用してローカルで音声認識を行う場合は、Rustをインストールしてください。[ここ](https://www.rust-lang.org/tools/install)に記載されている手順に従ってください。
+
+### 01 CLIのインストールと実行
+
+```shell
+pip install 01OS
+```
+
+```shell
+01 --server # ハードウェアデバイスがリスンするサーバーを起動します。
+```
+
+# クライアントのセットアップ
+
+### ESP32ボード用
+
+[ESP32セットアップドキュメント](https://github.com/OpenInterpreter/01/tree/main/01OS/01OS/clients/esp32)をご覧ください。
+
+### Mac、Windows、Ubuntuマシン用
+
+```
+01 # サーバーとクライアントを起動します。
+
+01 --server --expose # サーバーを起動し、Ngrok経由で公開します。クライアントが接続するための`server_url`が表示されます。
+
+01 --client --server_url your-server.com # クライアントのみを起動します。
+```
+
+### サービスプロバイダーの切り替え
+
+01は、音声認識、音声合成、言語モデルのプロバイダーに依存しません。
+
+以下のコマンドを実行して、プロバイダーを選択します。
+
+```shell
+01 --tts-service openai
+01 --llm-service openai
+01 --stt-service openai
+```
+
+[すべてのプロバイダーを見る ↗](https://docs.litellm.ai/docs/providers/)、または[サービスプロバイダーを追加して01チームに参加する。↗]()
+
+### 01をローカルで実行する
+
+一部のサービスプロバイダーはインターネット接続を必要としません。
+
+次のコマンドを実行すると、ハードウェアに最適なプロバイダーをダウンロードして使用しようとします。
+
+```shell
+01 --local
+```
+
+## 仕組み
+
+01は、言語モデル(音声インターフェースでラップされている)に`exec()`関数を装備し、コードを書いて実行してコンピューターを制御できるようにします。
+
+音声はエンドユーザーのデバイスとの間でのみストリーミングされます。
+
+# 貢献
+
+詳細については、[コントリビューションガイドライン](docs/CONTRIBUTING.md)をご覧ください。
+
+### 開発のためのセットアップ
+
+```bash
+# リポジトリをクローン
+git clone https://github.com/KillianLucas/01.git
+
+# 01OSディレクトリに移動
+cd 01OS
+
+# Pythonの依存関係をインストール
+poetry install
+
+# 実行
+poetry run 01
+```
+
+
+
+# ロードマップ
+
+01の未来を見るには、[私たちのロードマップ](https://github.com/KillianLucas/open-interpreter/blob/main/docs/ROADMAP.md)をご覧ください。
+
+
+
+## 背景
+
+### [コンテキスト ↗](https://github.com/KillianLucas/01/blob/main/CONTEXT.md)
+
+01以前のデバイスの物語。
+
+### [インスピレーション ↗](https://github.com/KillianLucas/01/tree/main/INSPIRATION.md)
+
+素晴らしいアイデアを盗みたいもの。
+
+
+
+## 方向性
+
+### [目標 ↗](https://github.com/KillianLucas/01/blob/main/GOALS.md)
+
+私たちがやろうとしていること。
+
+### [ユースケース ↗](https://github.com/KillianLucas/01/blob/main/USE_CASES.md)
+
+01ができるようになること。
+
+
\ No newline at end of file
From a559c4272124170d18195f0612352c7971652cb0 Mon Sep 17 00:00:00 2001
From: Maki
Date: Wed, 20 Mar 2024 01:14:48 +0900
Subject: [PATCH 02/39] Update README.md
---
README.md | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/README.md b/README.md
index 56d43bb..97f4a09 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,13 @@ Official pre-release repository for [The 01 Project](https://twitter.com/helloki
+
+
+ | [日本語](docs/README_JP.md) | [English](README.md) |
+
+
+
+
![poster](https://pbs.twimg.com/media/GDqTVYzbgAIfLJf?format=png&name=4096x4096)
From be1193267a9d8beeb27e3cac42bfba2460cc9885 Mon Sep 17 00:00:00 2001
From: Maki
Date: Wed, 20 Mar 2024 01:15:23 +0900
Subject: [PATCH 03/39] Update README_JP.md
---
docs/README_JP.md | 4 ----
1 file changed, 4 deletions(-)
diff --git a/docs/README_JP.md b/docs/README_JP.md
index bd7c689..ceda391 100644
--- a/docs/README_JP.md
+++ b/docs/README_JP.md
@@ -1,7 +1,3 @@
-はい、以下が自然な日本語に変換した文章になります。
-
-* * *
-
[The 01 Project](https://twitter.com/hellokillian/status/1745875973583896950)の公式プレリリースリポジトリ。
> 発売まで残り**3**日
From 17e1c3ef3b06da4d7ead6f77b988e9df7e87e8e0 Mon Sep 17 00:00:00 2001
From: Tasha Upchurch
Date: Tue, 26 Mar 2024 14:24:11 -0400
Subject: [PATCH 04/39] Added documentation links for youtube videos explaining
installation and usage
---
docs/video_documentation/collection.md | 83 ++++++++++++++++++++++++++
1 file changed, 83 insertions(+)
create mode 100644 docs/video_documentation/collection.md
diff --git a/docs/video_documentation/collection.md b/docs/video_documentation/collection.md
new file mode 100644
index 0000000..c0390c4
--- /dev/null
+++ b/docs/video_documentation/collection.md
@@ -0,0 +1,83 @@
+
+## For End Users
+[Announcment video](https://www.youtube.com/watch?v=jWr-WeXAdeI)
+[Wes Roth](https://www.youtube.com/@WesRoth)
+
+
+Details
+
+No technical coverage
+
+
+
+---
+
+[Announcment video](https://www.youtube.com/watch?v=JaBFT3fF2fk)
+[TheAIGRID](https://www.youtube.com/@TheAiGrid)
+
+
+Details
+
+[here](https://youtu.be/JaBFT3fF2fk?si=8zPGO-U6WdLNnISw&t=656)
+mentions the current lack of windows support
+
+
+
+---
+
+[Announcment video](https://www.youtube.com/watch?v=Q_p82HtBqoc)
+[Matt Berman](https://www.youtube.com/@matthew_berman)
+
+
+Details
+
+[here](https://youtu.be/Q_p82HtBqoc?si=aAxjWZnBdwBbaOUr&t=579)
+Berman shows an install of 01 using conda and python 3.9
+in.. looks like linux.. shows how to get openai keys.
+
+
+
+---
+
+[Announcment video](https://www.youtube.com/watch?v=q0dJ7T7au2Y)
+[WorldofAI](https://www.youtube.com/@intheworldofai)
+
+
+Details
+
+
+
+
+
+---
+
+[Breakdown video](https://www.youtube.com/watch?v=W-VwN0n4d9Y)
+[Mervin Praison](https://www.youtube.com/@MervinPraison)
+
+Details
+- uses conda to install 01 and uses python 3.11 on linux.. maybe mac
+- 0:00 Introduction to Open Interpreter
+- 0:47 Creating Apps and Summarizing Documents
+- 1:20 Image Modifications and Game Creation
+- 2:55 Exploratory Data Analysis and Charting
+- 4:00 Server Log Analysis
+- 5:01 Image and Video Editing
+- 6:00 Composing Music with AI
+- 7:18 Calendar Management and Email Automation
+- 9:01 Integrating with Fast API and LM Studio
+
+
+
+---
+
+[Breakdown video](https://www.youtube.com/watch?v=uyfoHQVgeY0)
+[Gary Explains](https://www.youtube.com/@GaryExplains)
+
for **open interpreter** not **01**
+
+Details
+- 3:45 states that it will run on mac/linux and windows and requires python 3.10
+
+
+## For Developers
+
+Coming soon
\ No newline at end of file
From 991cba8d09ca6731ea4e930984fd12aa3b8539e3 Mon Sep 17 00:00:00 2001
From: Robert Brisita <986796+rbrisita@users.noreply.github.com>
Date: Wed, 10 Apr 2024 14:53:16 -0400
Subject: [PATCH 05/39] Adding loglevel panic to suppress FFmpeg logs when
running STT locally.
---
software/source/server/services/stt/local-whisper/stt.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/software/source/server/services/stt/local-whisper/stt.py b/software/source/server/services/stt/local-whisper/stt.py
index 1c2743b..a92a703 100644
--- a/software/source/server/services/stt/local-whisper/stt.py
+++ b/software/source/server/services/stt/local-whisper/stt.py
@@ -156,7 +156,7 @@ def stt_wav(service_directory, wav_file_path: str):
temp_dir, f"output_stt_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav"
)
ffmpeg.input(wav_file_path).output(
- output_path, acodec="pcm_s16le", ac=1, ar="16k"
+ output_path, acodec="pcm_s16le", ac=1, ar="16k", loglevel="panic"
).run()
try:
transcript = get_transcription_file(service_directory, output_path)
From b24d48fd89b77a5bf5df5100c88061bc83a69462 Mon Sep 17 00:00:00 2001
From: Robert Brisita <986796+rbrisita@users.noreply.github.com>
Date: Wed, 10 Apr 2024 15:02:50 -0400
Subject: [PATCH 06/39] Adding check to true on subprocess run calls to expose
command errors that would otherwise fail silently.
---
software/source/server/services/stt/local-whisper/stt.py | 2 +-
software/source/server/services/stt/openai/stt.py | 2 +-
software/source/server/utils/bytes_to_wav.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/software/source/server/services/stt/local-whisper/stt.py b/software/source/server/services/stt/local-whisper/stt.py
index 1c2743b..d0719b7 100644
--- a/software/source/server/services/stt/local-whisper/stt.py
+++ b/software/source/server/services/stt/local-whisper/stt.py
@@ -125,7 +125,7 @@ def export_audio_to_wav_ffmpeg(audio: bytearray, mime_type: str) -> str:
def run_command(command):
result = subprocess.run(
- command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+ command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True
)
return result.stdout, result.stderr
diff --git a/software/source/server/services/stt/openai/stt.py b/software/source/server/services/stt/openai/stt.py
index 4cb1e4b..32dc9e2 100644
--- a/software/source/server/services/stt/openai/stt.py
+++ b/software/source/server/services/stt/openai/stt.py
@@ -70,7 +70,7 @@ def export_audio_to_wav_ffmpeg(audio: bytearray, mime_type: str) -> str:
def run_command(command):
result = subprocess.run(
- command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+ command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True
)
return result.stdout, result.stderr
diff --git a/software/source/server/utils/bytes_to_wav.py b/software/source/server/utils/bytes_to_wav.py
index a189257..a789792 100644
--- a/software/source/server/utils/bytes_to_wav.py
+++ b/software/source/server/utils/bytes_to_wav.py
@@ -57,7 +57,7 @@ def export_audio_to_wav_ffmpeg(audio: bytearray, mime_type: str) -> str:
def run_command(command):
result = subprocess.run(
- command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+ command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True
)
return result.stdout, result.stderr
From 3fbf2aa36223ab806e8e367adbfbb991550d4be2 Mon Sep 17 00:00:00 2001
From: Abdullah Gohar
Date: Fri, 12 Apr 2024 09:03:48 +0500
Subject: [PATCH 07/39] Fixed the ctrl-c bug
---
software/source/clients/base_device.py | 27 +++++++++++++++----
software/source/server/utils/process_utils.py | 6 ++++-
2 files changed, 27 insertions(+), 6 deletions(-)
diff --git a/software/source/clients/base_device.py b/software/source/clients/base_device.py
index 3bf900e..a614498 100644
--- a/software/source/clients/base_device.py
+++ b/software/source/clients/base_device.py
@@ -72,6 +72,7 @@ class Device:
self.captured_images = []
self.audiosegments = []
self.server_url = ""
+ self.ctrl_pressed = False
def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX):
"""Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list."""
@@ -256,23 +257,39 @@ class Device:
def on_press(self, key):
"""Detect spacebar press and Ctrl+C combination."""
self.pressed_keys.add(key) # Add the pressed key to the set
+
if keyboard.Key.space in self.pressed_keys:
self.toggle_recording(True)
- elif {keyboard.Key.ctrl, keyboard.KeyCode.from_char("c")} <= self.pressed_keys:
+ elif {keyboard.Key.ctrl, keyboard.KeyCode.from_char('c')} <= self.pressed_keys:
logger.info("Ctrl+C pressed. Exiting...")
kill_process_tree()
os._exit(0)
+
+ # Windows alternative to the above
+ if key == keyboard.Key.ctrl_l:
+ self.ctrl_pressed = True
+
+ try:
+ if key.vk == 67 and self.ctrl_pressed:
+ logger.info("Ctrl+C pressed. Exiting...")
+ kill_process_tree()
+ os._exit(0)
+ # For non-character keys
+ except:
+ pass
+
+
def on_release(self, key):
"""Detect spacebar release and 'c' key press for camera, and handle key release."""
- self.pressed_keys.discard(
- key
- ) # Remove the released key from the key press tracking set
+ self.pressed_keys.discard(key) # Remove the released key from the key press tracking set
+ if key == keyboard.Key.ctrl_l:
+ self.ctrl_pressed = False
if key == keyboard.Key.space:
self.toggle_recording(False)
- elif CAMERA_ENABLED and key == keyboard.KeyCode.from_char("c"):
+ elif CAMERA_ENABLED and key == keyboard.KeyCode.from_char('c'):
self.fetch_image_from_camera()
async def message_sender(self, websocket):
diff --git a/software/source/server/utils/process_utils.py b/software/source/server/utils/process_utils.py
index 586e4c6..5337bae 100644
--- a/software/source/server/utils/process_utils.py
+++ b/software/source/server/utils/process_utils.py
@@ -7,7 +7,11 @@ def kill_process_tree():
pid = os.getpid() # Get the current process ID
try:
# Send SIGTERM to the entire process group to ensure all processes are targeted
- os.killpg(os.getpgid(pid), signal.SIGKILL)
+ try:
+ os.killpg(os.getpgid(pid), signal.SIGKILL)
+ # Windows implementation
+ except AttributeError:
+ os.kill(pid, signal.SIGTERM)
parent = psutil.Process(pid)
children = parent.children(recursive=True)
for child in children:
From f365b6a515b3b1b5125277a66256cccd551400e3 Mon Sep 17 00:00:00 2001
From: Abdullah Gohar
Date: Sun, 14 Apr 2024 07:50:03 +0500
Subject: [PATCH 08/39] Fixed non-connection issue
---
software/source/clients/base_device.py | 15 +++++++++++++--
software/start.py | 6 ++++++
2 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/software/source/clients/base_device.py b/software/source/clients/base_device.py
index a614498..5d60c50 100644
--- a/software/source/clients/base_device.py
+++ b/software/source/clients/base_device.py
@@ -3,6 +3,7 @@ from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
import os
+import sys
import asyncio
import threading
import pyaudio
@@ -58,7 +59,17 @@ CAMERA_WARMUP_SECONDS = float(os.getenv("CAMERA_WARMUP_SECONDS", 0))
# Specify OS
current_platform = get_system_info()
-is_win10 = lambda: platform.system() == "Windows" and "10" in platform.version()
+
+def is_win11():
+ return sys.getwindowsversion().build >= 22000
+
+def is_win10():
+ try:
+ return platform.system() == "Windows" and "10" in platform.version() and not is_win11()
+ except:
+ return False
+
+print(platform.system(), platform.version())
# Initialize PyAudio
p = pyaudio.PyAudio()
@@ -359,7 +370,7 @@ class Device:
code = message["content"]
result = interpreter.computer.run(language, code)
send_queue.put(result)
-
+
if is_win10():
logger.info("Windows 10 detected")
# Workaround for Windows 10 not latching to the websocket server.
diff --git a/software/start.py b/software/start.py
index 4f3377f..6016d17 100644
--- a/software/start.py
+++ b/software/start.py
@@ -122,6 +122,10 @@ def _run(
# llm_service = "llamafile"
stt_service = "local-whisper"
select_local_model()
+
+ system_type = platform.system()
+ if system_type == "Windows":
+ server_host = "localhost"
if not server_url:
server_url = f"{server_host}:{server_port}"
@@ -129,6 +133,8 @@ def _run(
if not server and not client:
server = True
client = True
+
+
def handle_exit(signum, frame):
os._exit(0)
From 325fcea15bb8d2b1cf2cbd3ff98a39a0d17d7666 Mon Sep 17 00:00:00 2001
From: Abdullah Gohar
Date: Sun, 14 Apr 2024 07:58:16 +0500
Subject: [PATCH 09/39] Removed redundant logging lines
---
software/source/clients/base_device.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/software/source/clients/base_device.py b/software/source/clients/base_device.py
index 5d60c50..087ca55 100644
--- a/software/source/clients/base_device.py
+++ b/software/source/clients/base_device.py
@@ -69,7 +69,6 @@ def is_win10():
except:
return False
-print(platform.system(), platform.version())
# Initialize PyAudio
p = pyaudio.PyAudio()
From 471b5f8a8440123f1a40726c2e7f38f13ad348bf Mon Sep 17 00:00:00 2001
From: Robert Brisita <986796+rbrisita@users.noreply.github.com>
Date: Thu, 4 Apr 2024 17:40:46 -0400
Subject: [PATCH 10/39] Adding checks for dmesg log access and creation of it
if not accessible.
---
software/source/server/utils/kernel.py | 32 +++++++++++++++++++++++++-
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/software/source/server/utils/kernel.py b/software/source/server/utils/kernel.py
index fcca107..d3ce75c 100644
--- a/software/source/server/utils/kernel.py
+++ b/software/source/server/utils/kernel.py
@@ -5,12 +5,17 @@ load_dotenv() # take environment variables from .env.
import asyncio
import subprocess
import platform
+import os
+import shutil
from .logs import setup_logging
from .logs import logger
setup_logging()
+# dmesg process created at boot time
+dmesg_proc = None
+
def get_kernel_messages():
"""
@@ -25,12 +30,37 @@ def get_kernel_messages():
output, _ = process.communicate()
return output.decode("utf-8")
elif current_platform == "Linux":
- with open("/var/log/dmesg", "r") as file:
+ log_path = get_dmesg_log_path()
+ with open(log_path, 'r') as file:
return file.read()
else:
logger.info("Unsupported platform.")
+def get_dmesg_log_path():
+ """
+ Check for the existence of a readable dmesg log file and return its path.
+ Create an accessible path if not found.
+ """
+ if os.access('/var/log/dmesg', os.F_OK | os.R_OK):
+ return '/var/log/dmesg'
+
+ global dmesg_proc
+ dmesg_log_path = '/tmp/dmesg'
+ if dmesg_proc:
+ return dmesg_log_path
+
+ logger.info("Created /tmp/dmesg.")
+ subprocess.run(['touch', dmesg_log_path])
+ dmesg_path = shutil.which('dmesg')
+ if dmesg_path:
+ logger.info(f"Writing to {dmesg_log_path} from dmesg.")
+ dmesg_proc = subprocess.Popen([dmesg_path, '--follow'], text=True, stdout=subprocess.PIPE)
+ subprocess.Popen(['tee', dmesg_log_path], text=True, stdin=dmesg_proc.stdout, stdout=subprocess.DEVNULL)
+
+ return dmesg_log_path
+
+
def custom_filter(message):
# Check for {TO_INTERPRETER{ message here }TO_INTERPRETER} pattern
if "{TO_INTERPRETER{" in message and "}TO_INTERPRETER}" in message:
From 3fcd0fb2386f31b3dd8510daff488871736f2969 Mon Sep 17 00:00:00 2001
From: Ikko Eltociear Ashimine
Date: Tue, 16 Apr 2024 15:50:02 +0900
Subject: [PATCH 11/39] Add README_JA.md
---
docs/README_JA.md | 155 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 155 insertions(+)
create mode 100644 docs/README_JA.md
diff --git a/docs/README_JA.md b/docs/README_JA.md
new file mode 100644
index 0000000..f49a56c
--- /dev/null
+++ b/docs/README_JA.md
@@ -0,0 +1,155 @@
+○
+
+
+
+
+
+ オープンソースの言語モデルコンピュータ。
+
Light の予約 | 最新情報 | ドキュメント
+
+
+
+
+![OI-O1-BannerDemo-2](https://www.openinterpreter.com/OI-O1-BannerDemo-3.jpg)
+
+あなたのビルドをサポートします。[1対1のサポートを申し込む。](https://0ggfznkwh4j.typeform.com/to/kkStE8WF)
+
+
+
+> [!IMPORTANT]
+> この実験的なプロジェクトは急速に開発が進んでおり、基本的な安全策が欠けています。安定した `1.0` リリースまでは、機密情報や有料サービスへのアクセスがないデバイスでのみこのリポジトリを実行してください。
+>
+> **これらの懸念やその他の懸念に対処するための大幅な書き換えが[ここ](https://github.com/KillianLucas/01-rewrite/tree/main)で行われています。**
+
+
+
+**01 プロジェクト** は、AI 機器のためのオープンソースのエコシステムを構築しています。
+
+私たちの主力オペレーティングシステムは、Rabbit R1、Humane Pin、[Star Trek computer](https://www.youtube.com/watch?v=1ZXugicgn6U) のような会話デバイスを動かすことができます。
+
+私たちは、オープンでモジュラーでフリーであり続けることで、この分野の GNU/Linux になるつもりです。
+
+
+
+# ソフトウェア
+
+```shell
+git clone https://github.com/OpenInterpreter/01 # リポジトリのクローン
+cd 01/software # CD でソースディレクトリに移動
+```
+
+
+
+```shell
+brew install portaudio ffmpeg cmake # Mac OSXの依存関係のインストール
+poetry install # Pythonの依存関係のインストール
+export OPENAI_API_KEY=sk... # または、`poetry run 01 --local` を実行し、ローカルですべてを実行
+poetry run 01 # 01 Light シミュレーターを作動させる(スペースバーを押しながら話し、放す)
+```
+
+
+
+
+
+# ハードウェア
+
+- **01 Light** は ESP32 ベースの音声インターフェースです。ビルド手順は[こちら](https://github.com/OpenInterpreter/01/tree/main/hardware/light)。買うべきもののリストは[こちら](https://github.com/OpenInterpreter/01/blob/main/hardware/light/BOM.md)。
+- ご自宅のコンピューターで動作している **01 サーバー**([下記のセットアップガイド](https://github.com/OpenInterpreter/01/blob/main/README.md#01-server))と連動して動作します。
+- **Mac OSX** と **Ubuntu** は `poetry run 01` を実行することでサポートされます(**Windows** は実験的にサポートされている)。これはスペースキーを使って 01 Light をシミュレートします。
+- (近日発表) **01 Heavy** は、ローカルですべてを実行するスタンドアローンデバイスです。
+
+**より多くのハードウェアをサポートし、構築するためには、皆さんの協力が必要です。** 01 は、入力(マイク、キーボードなど)、出力(スピーカー、スクリーン、モーターなど)、インターネット接続(またはローカルですべてを実行するのに十分な計算能力)があれば、どのようなデバイスでも実行できるはずです。[コントリビューションガイド →](https://github.com/OpenInterpreter/01/blob/main/CONTRIBUTING.md)
+
+
+
+# 何をするのか?
+
+01 は、`localhost:10001` で音声合成ウェブソケットを公開しています。
+
+生のオーディオバイトを[ストリーミング LMC フォーマット](https://docs.openinterpreter.com/guides/streaming-response)で `/` にストリーミングすると、同じフォーマットで応答を受け取ります。
+
+[Andrej Karpathy の LLM OS](https://twitter.com/karpathy/status/1723140519554105733) に一部インスパイアされ、[コード解釈言語モデル](https://github.com/OpenInterpreter/open-interpreter)を実行し、コンピュータの[カーネル](https://github.com/OpenInterpreter/01/blob/main/software/source/server/utils/kernel.py)で特定のイベントが発生したときにそれを呼び出します。
+
+01 はこれを音声インターフェースで包んでいます:
+
+
+
+
+
+# プロトコル
+
+## LMC メッセージ
+
+このシステムのさまざまなコンポーネントと通信するために、[LMC メッセージ](https://docs.openinterpreter.com/protocols/lmc-messages)フォーマットを導入します。これは、OpenAI のメッセージフォーマットを拡張し、"computer" の役割を含むようにしたものです:
+
+https://github.com/OpenInterpreter/01/assets/63927363/8621b075-e052-46ba-8d2e-d64b9f2a5da9
+
+## ダイナミックシステムメッセージ
+
+ダイナミックシステムメッセージは、LLM のシステムメッセージが AI に表示される一瞬前に、その中でコードを実行することを可能にします。
+
+```python
+# i.py の以下の設定を編集
+interpreter.system_message = r" The time is {{time.time()}}. " # 二重括弧の中は Python として実行されます
+interpreter.chat("What time is it?") # ツール/API を呼び出すことなく、次のことが分かります
+```
+
+# ガイド
+
+## 01 サーバー
+
+デスクトップ上でサーバーを起動し、01 Light に接続するには、以下のコマンドを実行します:
+
+```shell
+brew install ngrok/ngrok/ngrok
+ngrok authtoken ... # ngrok authtoken を使用
+poetry run 01 --server --expose
+```
+
+最後のコマンドは、サーバーの URL を表示します。これを 01 Light のキャプティブ WiFi ポータルに入力すると、01 Server に接続できます。
+
+## ローカルモード
+
+```
+poetry run 01 --local
+```
+
+Whisper を使ってローカル音声合成を実行したい場合、Rust をインストールする必要があります。[こちら](https://www.rust-lang.org/tools/install)の指示に従ってください。
+
+## カスタマイズ
+
+システムの動作をカスタマイズするには、`i.py` 内の[システムメッセージ、モデル、スキルライブラリのパス](https://docs.openinterpreter.com/settings/all-settings)などを編集します。このファイルはインタープリターをセットアップするもので、Open Interpreter によって動作します。
+
+## Ubuntu 依存関係
+
+```bash
+sudo apt-get install portaudio19-dev ffmpeg cmake
+```
+
+# コントリビューター
+
+[![01 project contributors](https://contrib.rocks/image?repo=OpenInterpreter/01&max=2000)](https://github.com/OpenInterpreter/01/graphs/contributors)
+
+参加方法の詳細については、[コントリビューションガイド](CONTRIBUTING.md)をご覧ください。
+
+
+
+# ロードマップ
+
+01 の未来を見るには、[私達のロードマップ](/ROADMAP.md)をご覧ください。
+
+
+
+## バックグラウンド
+
+### [コンテキスト ↗](https://github.com/KillianLucas/01/blob/main/CONTEXT.md)
+
+01 以前のデバイスの物語。
+
+### [インスピレーション ↗](https://github.com/KillianLucas/01/tree/main/INSPIRATION.md)
+
+素晴らしいアイデアは盗みたいと思うもの。
+
+
+
+○
From 59e32562a8bb6bc63c371459ca31ea76dfc6045b Mon Sep 17 00:00:00 2001
From: Ikko Eltociear Ashimine
Date: Tue, 16 Apr 2024 15:51:15 +0900
Subject: [PATCH 12/39] Update README_JA.md
---
docs/README_JA.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/README_JA.md b/docs/README_JA.md
index f49a56c..868dae8 100644
--- a/docs/README_JA.md
+++ b/docs/README_JA.md
@@ -130,7 +130,7 @@ sudo apt-get install portaudio19-dev ffmpeg cmake
[![01 project contributors](https://contrib.rocks/image?repo=OpenInterpreter/01&max=2000)](https://github.com/OpenInterpreter/01/graphs/contributors)
-参加方法の詳細については、[コントリビューションガイド](CONTRIBUTING.md)をご覧ください。
+参加方法の詳細については、[コントリビューションガイド](/CONTRIBUTING.md)をご覧ください。
From e397f8819f37d88e7af320ba1c7b5c6788c36ead Mon Sep 17 00:00:00 2001
From: Ben Xu
Date: Wed, 24 Apr 2024 12:54:18 -0400
Subject: [PATCH 13/39] add binary audio parsing
---
.../ios/react-native/package-lock.json | 13 +++
.../clients/ios/react-native/package.json | 8 +-
.../ios/react-native/src/screens/Main.tsx | 99 +++++++++++++++----
software/source/server/server.py | 18 +++-
software/source/server/tunnel.py | 2 +-
software/source/utils/accumulator.py | 4 +-
6 files changed, 115 insertions(+), 29 deletions(-)
diff --git a/software/source/clients/ios/react-native/package-lock.json b/software/source/clients/ios/react-native/package-lock.json
index 81e2031..97b3864 100644
--- a/software/source/clients/ios/react-native/package-lock.json
+++ b/software/source/clients/ios/react-native/package-lock.json
@@ -17,12 +17,14 @@
"expo-status-bar": "~1.11.1",
"react": "18.2.0",
"react-native": "0.73.4",
+ "react-native-base64": "^0.2.1",
"react-native-safe-area-context": "4.8.2",
"react-native-screens": "~3.29.0"
},
"devDependencies": {
"@babel/core": "^7.20.0",
"@types/react": "~18.2.45",
+ "@types/react-native-base64": "^0.2.2",
"typescript": "^5.1.3"
}
},
@@ -6102,6 +6104,12 @@
"csstype": "^3.0.2"
}
},
+ "node_modules/@types/react-native-base64": {
+ "version": "0.2.2",
+ "resolved": "https://registry.npmjs.org/@types/react-native-base64/-/react-native-base64-0.2.2.tgz",
+ "integrity": "sha512-obr+/L9Jaxdr+xCVS/IQcYgreg5xtnui4Wqw/G1acBUtW2CnqVJj6lK6F/5F3+5d2oZEo5xDDLqy8GVn2HbEmw==",
+ "dev": true
+ },
"node_modules/@types/scheduler": {
"version": "0.16.8",
"resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz",
@@ -11492,6 +11500,11 @@
"react": "18.2.0"
}
},
+ "node_modules/react-native-base64": {
+ "version": "0.2.1",
+ "resolved": "https://registry.npmjs.org/react-native-base64/-/react-native-base64-0.2.1.tgz",
+ "integrity": "sha512-eHgt/MA8y5ZF0aHfZ1aTPcIkDWxza9AaEk4GcpIX+ZYfZ04RcaNahO+527KR7J44/mD3efYfM23O2C1N44ByWA=="
+ },
"node_modules/react-native-safe-area-context": {
"version": "4.8.2",
"resolved": "https://registry.npmjs.org/react-native-safe-area-context/-/react-native-safe-area-context-4.8.2.tgz",
diff --git a/software/source/clients/ios/react-native/package.json b/software/source/clients/ios/react-native/package.json
index c031609..86faf84 100644
--- a/software/source/clients/ios/react-native/package.json
+++ b/software/source/clients/ios/react-native/package.json
@@ -13,18 +13,20 @@
"@react-navigation/native": "^6.1.14",
"@react-navigation/native-stack": "^6.9.22",
"expo": "~50.0.8",
+ "expo-av": "~13.10.5",
+ "expo-barcode-scanner": "~12.9.3",
"expo-camera": "~14.0.5",
"expo-status-bar": "~1.11.1",
"react": "18.2.0",
"react-native": "0.73.4",
+ "react-native-base64": "^0.2.1",
"react-native-safe-area-context": "4.8.2",
- "react-native-screens": "~3.29.0",
- "expo-barcode-scanner": "~12.9.3",
- "expo-av": "~13.10.5"
+ "react-native-screens": "~3.29.0"
},
"devDependencies": {
"@babel/core": "^7.20.0",
"@types/react": "~18.2.45",
+ "@types/react-native-base64": "^0.2.2",
"typescript": "^5.1.3"
},
"ios": {
diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx
index 3823c43..966c8af 100644
--- a/software/source/clients/ios/react-native/src/screens/Main.tsx
+++ b/software/source/clients/ios/react-native/src/screens/Main.tsx
@@ -1,6 +1,9 @@
import React, { useState, useEffect } from "react";
import { View, Text, TouchableOpacity, StyleSheet } from "react-native";
-import { Audio } from "expo-av";
+import * as FileSystem from 'expo-file-system';
+import { AVPlaybackStatus, Audio } from "expo-av";
+import { Buffer } from "buffer";
+import base64 from 'react-native-base64';
interface MainProps {
route: {
@@ -18,39 +21,83 @@ const Main: React.FC = ({ route }) => {
const [ws, setWs] = useState(null);
const [recording, setRecording] = useState(null);
const [audioQueue, setAudioQueue] = useState([]);
+ const [isPlaying, setIsPlaying] = useState(false);
+ const Buffer = require('buffer/').Buffer;
+
+ const constructTempFilePath = async (buffer: Buffer) => {
+ const tempFilePath = `${FileSystem.cacheDirectory}${Date.now()}` + "speech.mp3";
+ await FileSystem.writeAsStringAsync(
+ tempFilePath,
+ buffer.toString("base64"),
+ {
+ encoding: FileSystem.EncodingType.Base64,
+ }
+ );
- useEffect(() => {
- const playNextAudio = async () => {
- if (audioQueue.length > 0) {
- const uri = audioQueue.shift();
- const { sound } = await Audio.Sound.createAsync(
- { uri: uri! },
- { shouldPlay: true }
- );
- sound.setOnPlaybackStatusUpdate(async (status) => {
- if (status.didJustFinish && !status.isLooping) {
- await sound.unloadAsync();
- playNextAudio();
- }
- });
+ return tempFilePath;
+ };
+
+ const playNextAudio = async () => {
+ console.log("in playNextAudio audioQueue is", audioQueue);
+ console.log("isPlaying is", isPlaying);
+
+ if (audioQueue.length > 0) {
+ const uri = audioQueue.shift() as string;
+ console.log("load audio from", uri);
+ setIsPlaying(true);
+
+ try {
+ const { sound } = await Audio.Sound.createAsync({ uri });
+ await sound.playAsync();
+ console.log("playing audio from", uri);
+
+ sound.setOnPlaybackStatusUpdate(_onPlaybackStatusUpdate);
+ } catch (error){
+ console.log("Error playing audio", error);
+ setIsPlaying(false);
+ playNextAudio();
}
- };
+ }
+ };
+
+ const _onPlaybackStatusUpdate = (status: AVPlaybackStatus) => {
+ if (status.isLoaded && status.didJustFinish) {
+ setIsPlaying(false);
+ playNextAudio();
+ }
+ };
+
+ useEffect(() => {
let websocket: WebSocket;
try {
console.log("Connecting to WebSocket at " + scannedData);
websocket = new WebSocket(scannedData);
+ websocket.binaryType = "blob";
websocket.onopen = () => {
setConnectionStatus(`Connected to ${scannedData}`);
console.log("WebSocket connected");
};
+
websocket.onmessage = async (e) => {
- console.log("Received message: ", e.data);
- setAudioQueue((prevQueue) => [...prevQueue, e.data]);
- if (audioQueue.length === 1) {
+ const message = JSON.parse(e.data);
+
+ if (message.content) {
+
+ const parsedMessage = message.content.replace(/^b'|['"]|['"]$/g, "");
+ const buffer = Buffer.from(parsedMessage, 'base64')
+ console.log("parsed message", buffer.toString());
+
+ const uri = await constructTempFilePath(buffer);
+ setAudioQueue((prevQueue) => [...prevQueue, uri]);
+ }
+
+ if (message.format === "bytes.raw" && message.end) {
+ console.log("calling playNextAudio");
playNextAudio();
}
+
};
websocket.onerror = (error) => {
@@ -74,7 +121,7 @@ const Main: React.FC = ({ route }) => {
websocket.close();
}
};
- }, [scannedData, audioQueue]);
+ }, [scannedData]);
const startRecording = async () => {
if (recording) {
@@ -108,7 +155,17 @@ const Main: React.FC = ({ route }) => {
const uri = recording.getURI();
console.log("Recording stopped and stored at", uri);
if (ws && uri) {
- ws.send(uri);
+ const response = await fetch(uri);
+ const blob = await response.blob();
+ const reader = new FileReader();
+ reader.readAsArrayBuffer(blob);
+ reader.onloadend = () => {
+ const audioBytes = reader.result;
+ if (audioBytes) {
+ ws.send(audioBytes);
+ console.log("sent audio bytes to WebSocket");
+ }
+ };
}
}
};
diff --git a/software/source/server/server.py b/software/source/server/server.py
index c4dd036..444298d 100644
--- a/software/source/server/server.py
+++ b/software/source/server/server.py
@@ -20,6 +20,7 @@ from interpreter import interpreter
from ..utils.accumulator import Accumulator
from .utils.logs import setup_logging
from .utils.logs import logger
+import base64
from ..utils.print_markdown import print_markdown
@@ -194,13 +195,24 @@ async def receive_messages(websocket: WebSocket):
async def send_messages(websocket: WebSocket):
while True:
message = await to_device.get()
- # print(f"Sending to the device: {type(message)} {str(message)[:100]}")
try:
if isinstance(message, dict):
+ print(f"Sending to the device: {type(message)} {str(message)[:100]}")
await websocket.send_json(message)
elif isinstance(message, bytes):
- await websocket.send_bytes(message)
+ message = base64.b64encode(message)
+ str_bytes = str(message)
+ json_bytes = {
+ "role": "assistant",
+ "type": "audio",
+ "format": "message",
+ "content": str_bytes,
+ }
+ print(
+ f"Sending to the device: {type(json_bytes)} {str(json_bytes)[:100]}"
+ )
+ await websocket.send_json(json_bytes)
else:
raise TypeError("Message must be a dict or bytes")
except:
@@ -286,7 +298,7 @@ async def listener():
logger.debug("Got chunk:", chunk)
# Send it to the user
- await to_device.put(chunk)
+ # await to_device.put(chunk)
# Yield to the event loop, so you actually send it out
await asyncio.sleep(0.01)
diff --git a/software/source/server/tunnel.py b/software/source/server/tunnel.py
index 809db08..0e0ad17 100644
--- a/software/source/server/tunnel.py
+++ b/software/source/server/tunnel.py
@@ -100,7 +100,7 @@ def create_tunnel(
# If ngrok is installed, start it on the specified port
# process = subprocess.Popen(f'ngrok http {server_port} --log=stdout', shell=True, stdout=subprocess.PIPE)
process = subprocess.Popen(
- f"ngrok http {server_port} --scheme http,https --domain=marten-advanced-dragon.ngrok-free.app --log=stdout",
+ f"ngrok http {server_port} --scheme http,https --domain=sterling-snail-conversely.ngrok-free.app --log=stdout",
shell=True,
stdout=subprocess.PIPE,
)
diff --git a/software/source/utils/accumulator.py b/software/source/utils/accumulator.py
index 37912b5..9f66e89 100644
--- a/software/source/utils/accumulator.py
+++ b/software/source/utils/accumulator.py
@@ -44,4 +44,6 @@ class Accumulator:
if "content" not in self.message or type(self.message["content"]) != bytes:
self.message["content"] = b""
self.message["content"] += chunk
- return None
+ self.message["type"] = "audio"
+ self.message["format"] = "bytes.wav"
+ return self.message
From 163b28383cd3f49ea6d29e43be56f8c322e0a733 Mon Sep 17 00:00:00 2001
From: Ben Xu
Date: Wed, 24 Apr 2024 17:47:20 -0400
Subject: [PATCH 14/39] integrate working toy
---
.../ios/react-native/src/screens/Main.tsx | 97 +++++++++++--------
1 file changed, 54 insertions(+), 43 deletions(-)
diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx
index 966c8af..4bbde42 100644
--- a/software/source/clients/ios/react-native/src/screens/Main.tsx
+++ b/software/source/clients/ios/react-native/src/screens/Main.tsx
@@ -1,9 +1,7 @@
import React, { useState, useEffect } from "react";
import { View, Text, TouchableOpacity, StyleSheet } from "react-native";
import * as FileSystem from 'expo-file-system';
-import { AVPlaybackStatus, Audio } from "expo-av";
-import { Buffer } from "buffer";
-import base64 from 'react-native-base64';
+import { Audio } from "expo-av";
interface MainProps {
route: {
@@ -15,58 +13,58 @@ interface MainProps {
const Main: React.FC = ({ route }) => {
const { scannedData } = route.params;
-
- const [connectionStatus, setConnectionStatus] =
- useState("Connecting...");
+ const [connectionStatus, setConnectionStatus] = useState("Connecting...");
const [ws, setWs] = useState(null);
const [recording, setRecording] = useState(null);
const [audioQueue, setAudioQueue] = useState([]);
- const [isPlaying, setIsPlaying] = useState(false);
- const Buffer = require('buffer/').Buffer;
-
- const constructTempFilePath = async (buffer: Buffer) => {
- const tempFilePath = `${FileSystem.cacheDirectory}${Date.now()}` + "speech.mp3";
- await FileSystem.writeAsStringAsync(
- tempFilePath,
- buffer.toString("base64"),
- {
- encoding: FileSystem.EncodingType.Base64,
- }
- );
-
- return tempFilePath;
- };
+ const [sound, setSound] = useState();
+ const audioDir = FileSystem.documentDirectory + '01/audio/';
+
+
+ async function dirExists() {
+ /**
+ * Checks if audio directory exists in device storage, if not creates it.
+ */
+ const dirInfo = await FileSystem.getInfoAsync(audioDir);
+ if (!dirInfo.exists) {
+ console.log("audio directory doesn't exist, creating...");
+ await FileSystem.makeDirectoryAsync(audioDir, { intermediates: true });
+ }
+ }
const playNextAudio = async () => {
- console.log("in playNextAudio audioQueue is", audioQueue);
- console.log("isPlaying is", isPlaying);
+ await dirExists();
+ console.log("in playNextAudio audioQueue is", audioQueue.length);
if (audioQueue.length > 0) {
const uri = audioQueue.shift() as string;
console.log("load audio from", uri);
- setIsPlaying(true);
try {
const { sound } = await Audio.Sound.createAsync({ uri });
- await sound.playAsync();
- console.log("playing audio from", uri);
+ setSound(sound);
- sound.setOnPlaybackStatusUpdate(_onPlaybackStatusUpdate);
+ console.log("playing audio from", uri);
+ await sound?.playAsync();
} catch (error){
console.log("Error playing audio", error);
- setIsPlaying(false);
playNextAudio();
}
}
};
- const _onPlaybackStatusUpdate = (status: AVPlaybackStatus) => {
- if (status.isLoaded && status.didJustFinish) {
- setIsPlaying(false);
- playNextAudio();
- }
- };
+ useEffect(() => {
+ return sound
+ ? () => {
+ console.log('Unloading Sound');
+ sound.unloadAsync();
+ setSound(null);
+ playNextAudio();
+ }
+ : undefined;
+ }, [sound]);
+
useEffect(() => {
let websocket: WebSocket;
@@ -84,13 +82,21 @@ const Main: React.FC = ({ route }) => {
const message = JSON.parse(e.data);
if (message.content) {
-
const parsedMessage = message.content.replace(/^b'|['"]|['"]$/g, "");
- const buffer = Buffer.from(parsedMessage, 'base64')
- console.log("parsed message", buffer.toString());
+ console.log("parsedMessage", parsedMessage.slice(0, 30));
+
+ const filePath = `${audioDir}${Date.now()}.mp3`;
+ await FileSystem.writeAsStringAsync(
+ filePath,
+ parsedMessage,
+ {
+ encoding: FileSystem.EncodingType.Base64,
+ }
+ );
+
+ console.log("audio file written to", filePath);
- const uri = await constructTempFilePath(buffer);
- setAudioQueue((prevQueue) => [...prevQueue, uri]);
+ setAudioQueue((prevQueue) => [...prevQueue, filePath]);
}
if (message.format === "bytes.raw" && message.end) {
@@ -138,7 +144,7 @@ const Main: React.FC = ({ route }) => {
});
console.log("Starting recording..");
const { recording: newRecording } = await Audio.Recording.createAsync(
- Audio.RECORDING_OPTIONS_PRESET_HIGH_QUALITY
+ Audio.RecordingOptionsPresets.HIGH_QUALITY
);
setRecording(newRecording);
console.log("Recording started");
@@ -152,8 +158,12 @@ const Main: React.FC = ({ route }) => {
setRecording(null);
if (recording) {
await recording.stopAndUnloadAsync();
+ await Audio.setAudioModeAsync({
+ allowsRecordingIOS: false,
+ });
const uri = recording.getURI();
console.log("Recording stopped and stored at", uri);
+
if (ws && uri) {
const response = await fetch(uri);
const blob = await response.blob();
@@ -191,14 +201,15 @@ const Main: React.FC = ({ route }) => {
);
-};
+}
const styles = StyleSheet.create({
container: {
flex: 1,
- justifyContent: "center",
+ justifyContent: 'center',
alignItems: "center",
- backgroundColor: "#fff",
+ backgroundColor: '#ecf0f1',
+ padding: 10,
},
circle: {
width: 100,
From f673744f1be0d41f57e2d02223e55857f5ec7b50 Mon Sep 17 00:00:00 2001
From: Ben Xu
Date: Wed, 24 Apr 2024 20:46:36 -0400
Subject: [PATCH 15/39] upload file to gcloud storage
---
.../ios/react-native/src/screens/Main.tsx | 69 +++++++++++++++----
software/source/server/server.py | 20 ++++++
2 files changed, 76 insertions(+), 13 deletions(-)
diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx
index 4bbde42..d6a69cb 100644
--- a/software/source/clients/ios/react-native/src/screens/Main.tsx
+++ b/software/source/clients/ios/react-native/src/screens/Main.tsx
@@ -19,6 +19,37 @@ const Main: React.FC = ({ route }) => {
const [audioQueue, setAudioQueue] = useState([]);
const [sound, setSound] = useState();
const audioDir = FileSystem.documentDirectory + '01/audio/';
+ const Buffer = require('buffer').Buffer;
+
+ const toBuffer = async (blob: Blob) => {
+
+ const uri = await toDataURI(blob);
+ const base64 = uri.replace(/^.*,/g, "");
+ return Buffer.from(base64, "base64");
+ };
+
+ const toDataURI = (blob: Blob) =>
+ new Promise((resolve) => {
+ const reader = new FileReader();
+ reader.readAsDataURL(blob);
+ reader.onloadend = () => {
+ const uri = reader.result?.toString();
+ resolve(uri);
+ };
+ });
+
+ const constructTempFilePath = async (buffer: Buffer) => {
+ const tempFilePath = `${audioDir}${Date.now()}.wav`;
+ await FileSystem.writeAsStringAsync(
+ tempFilePath,
+ buffer.toString(),
+ {
+ encoding: FileSystem.EncodingType.Base64,
+ }
+ );
+
+ return tempFilePath;
+ };
async function dirExists() {
@@ -33,7 +64,6 @@ const Main: React.FC = ({ route }) => {
}
const playNextAudio = async () => {
- await dirExists();
console.log("in playNextAudio audioQueue is", audioQueue.length);
if (audioQueue.length > 0) {
@@ -65,6 +95,12 @@ const Main: React.FC = ({ route }) => {
: undefined;
}, [sound]);
+ useEffect(() => {
+ console.log("audioQueue has been updated:", audioQueue.length);
+ if (audioQueue.length == 1) {
+ playNextAudio();
+ }
+ }, [audioQueue]);
useEffect(() => {
let websocket: WebSocket;
@@ -79,31 +115,38 @@ const Main: React.FC = ({ route }) => {
};
websocket.onmessage = async (e) => {
+ console.log("Received message from WebSocket", e.data);
+
+ const blob = await e.data;
+ const buffer = await toBuffer(blob);
+ const filePath = await constructTempFilePath(buffer);
+ setAudioQueue((prevQueue) => [...prevQueue, filePath]);
+ console.log("audio file written to", filePath);
+
+ if (e.data.format === "bytes.raw" && e.data.end && audioQueue.length > 1) {
+ console.log("calling playNextAudio");
+ playNextAudio();
+ }
+
+ /**
const message = JSON.parse(e.data);
if (message.content) {
const parsedMessage = message.content.replace(/^b'|['"]|['"]$/g, "");
console.log("parsedMessage", parsedMessage.slice(0, 30));
- const filePath = `${audioDir}${Date.now()}.mp3`;
- await FileSystem.writeAsStringAsync(
- filePath,
- parsedMessage,
- {
- encoding: FileSystem.EncodingType.Base64,
- }
- );
-
- console.log("audio file written to", filePath);
-
+ const filePath = await constructFilePath(parsedMessage);
setAudioQueue((prevQueue) => [...prevQueue, filePath]);
+ console.log("audio file written to", filePath);
}
- if (message.format === "bytes.raw" && message.end) {
+ if (message.format === "bytes.raw" && message.end && audioQueue.length > 1) {
console.log("calling playNextAudio");
playNextAudio();
}
+ */
+
};
websocket.onerror = (error) => {
diff --git a/software/source/server/server.py b/software/source/server/server.py
index 444298d..2328f52 100644
--- a/software/source/server/server.py
+++ b/software/source/server/server.py
@@ -21,6 +21,7 @@ from ..utils.accumulator import Accumulator
from .utils.logs import setup_logging
from .utils.logs import logger
import base64
+from google.cloud import storage
from ..utils.print_markdown import print_markdown
@@ -202,6 +203,10 @@ async def send_messages(websocket: WebSocket):
await websocket.send_json(message)
elif isinstance(message, bytes):
message = base64.b64encode(message)
+ print(f"Sending to the device: {type(message)} {str(message)[:100]}")
+ await websocket.send_bytes(message)
+
+ """
str_bytes = str(message)
json_bytes = {
"role": "assistant",
@@ -213,6 +218,7 @@ async def send_messages(websocket: WebSocket):
f"Sending to the device: {type(json_bytes)} {str(json_bytes)[:100]}"
)
await websocket.send_json(json_bytes)
+ """
else:
raise TypeError("Message must be a dict or bytes")
except:
@@ -254,6 +260,7 @@ async def listener():
# Format will be bytes.wav or bytes.opus
mime_type = "audio/" + message["format"].split(".")[1]
audio_file_path = bytes_to_wav(message["content"], mime_type)
+ print("Audio file path:", audio_file_path)
# For microphone debugging:
if False:
@@ -387,6 +394,19 @@ def stream_tts(sentence):
with open(audio_file, "rb") as f:
audio_bytes = f.read()
+
+ storage_client = storage.Client(project="react-native-421323")
+ bucket = storage_client.bucket("01-audio")
+ blob = bucket.blob(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
+ generation_match_precondition = 0
+
+ blob.upload_from_filename(
+ audio_file, if_generation_match=generation_match_precondition
+ )
+ print(
+ f"Audio file {audio_file} uploaded to {datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav"
+ )
+
os.remove(audio_file)
file_type = "bytes.raw"
From 0602348f1c377445a4c648dca4bbb20f0a585934 Mon Sep 17 00:00:00 2001
From: Ty Fiero
Date: Wed, 24 Apr 2024 17:59:41 -0700
Subject: [PATCH 16/39] send wav files
---
.gitignore | 3 ++
software/source/server/server.py | 45 +++++++++----------
.../source/server/services/tts/openai/tts.py | 4 +-
software/source/server/tunnel.py | 2 +-
4 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/.gitignore b/.gitignore
index aeaed36..da03d3f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -169,3 +169,6 @@ cython_debug/
_.aifs
software/output_audio.wav
.DS_Store
+
+node_modules/
+.expo/
\ No newline at end of file
diff --git a/software/source/server/server.py b/software/source/server/server.py
index 2328f52..dd71b06 100644
--- a/software/source/server/server.py
+++ b/software/source/server/server.py
@@ -21,8 +21,7 @@ from ..utils.accumulator import Accumulator
from .utils.logs import setup_logging
from .utils.logs import logger
import base64
-from google.cloud import storage
-
+import shutil
from ..utils.print_markdown import print_markdown
os.environ["STT_RUNNER"] = "server"
@@ -394,31 +393,31 @@ def stream_tts(sentence):
with open(audio_file, "rb") as f:
audio_bytes = f.read()
-
- storage_client = storage.Client(project="react-native-421323")
- bucket = storage_client.bucket("01-audio")
- blob = bucket.blob(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
- generation_match_precondition = 0
-
- blob.upload_from_filename(
- audio_file, if_generation_match=generation_match_precondition
- )
- print(
- f"Audio file {audio_file} uploaded to {datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav"
- )
+ desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop')
+ desktop_audio_file = os.path.join(desktop_path, os.path.basename(audio_file))
+ shutil.copy(audio_file, desktop_audio_file)
+ print(f"Audio file saved to Desktop: {desktop_audio_file}")
+ # storage_client = storage.Client(project="react-native-421323")
+ # bucket = storage_client.bucket("01-audio")
+ # blob = bucket.blob(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
+ # generation_match_precondition = 0
+
+ # blob.upload_from_filename(
+ # audio_file, if_generation_match=generation_match_precondition
+ # )
+ # print(
+ # f"Audio file {audio_file} uploaded to {datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav"
+ # )
os.remove(audio_file)
- file_type = "bytes.raw"
- chunk_size = 1024
-
- # Stream the audio
- yield {"role": "assistant", "type": "audio", "format": file_type, "start": True}
- for i in range(0, len(audio_bytes), chunk_size):
- chunk = audio_bytes[i : i + chunk_size]
- yield chunk
- yield {"role": "assistant", "type": "audio", "format": file_type, "end": True}
+ file_type = "audio/wav"
+ # Read the entire WAV file
+ with open(audio_file, "rb") as f:
+ audio_bytes = f.read()
+ # Stream the audio as a single message
+ yield {"role": "assistant", "type": "audio", "format": file_type, "content": base64.b64encode(audio_bytes).decode('utf-8'), "start": True, "end": True}
from uvicorn import Config, Server
import os
diff --git a/software/source/server/services/tts/openai/tts.py b/software/source/server/services/tts/openai/tts.py
index 07e1eec..021353b 100644
--- a/software/source/server/services/tts/openai/tts.py
+++ b/software/source/server/services/tts/openai/tts.py
@@ -36,9 +36,9 @@ class Tts:
response.stream_to_file(temp_file.name)
# TODO: hack to format audio correctly for device
- outfile = tempfile.gettempdir() + "/" + "raw.dat"
+ outfile = tempfile.gettempdir() + "/" + "output.wav"
ffmpeg.input(temp_file.name).output(
- outfile, f="s16le", ar="16000", ac="1", loglevel="panic"
+ outfile, f="wav", ar="16000", ac="1", loglevel="panic"
).run()
return outfile
diff --git a/software/source/server/tunnel.py b/software/source/server/tunnel.py
index 0e0ad17..f25a0b3 100644
--- a/software/source/server/tunnel.py
+++ b/software/source/server/tunnel.py
@@ -100,7 +100,7 @@ def create_tunnel(
# If ngrok is installed, start it on the specified port
# process = subprocess.Popen(f'ngrok http {server_port} --log=stdout', shell=True, stdout=subprocess.PIPE)
process = subprocess.Popen(
- f"ngrok http {server_port} --scheme http,https --domain=sterling-snail-conversely.ngrok-free.app --log=stdout",
+ f"ngrok http {server_port} --scheme http,https --log=stdout",
shell=True,
stdout=subprocess.PIPE,
)
From 1e1db0836cd739923f63a037f2bc4e0f20ef4851 Mon Sep 17 00:00:00 2001
From: Ben Xu
Date: Wed, 24 Apr 2024 22:09:59 -0400
Subject: [PATCH 17/39] queue audio
---
.gitignore | 3 +-
.../ios/react-native/src/screens/Main.tsx | 63 +++++++++----------
software/source/server/server.py | 20 ++++--
3 files changed, 46 insertions(+), 40 deletions(-)
diff --git a/.gitignore b/.gitignore
index da03d3f..c0e95ec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -170,5 +170,6 @@ _.aifs
software/output_audio.wav
.DS_Store
+# ignore node modules and .expo files
node_modules/
-.expo/
\ No newline at end of file
+.expo/
diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx
index d6a69cb..a9a5ed3 100644
--- a/software/source/clients/ios/react-native/src/screens/Main.tsx
+++ b/software/source/clients/ios/react-native/src/screens/Main.tsx
@@ -1,7 +1,7 @@
import React, { useState, useEffect } from "react";
import { View, Text, TouchableOpacity, StyleSheet } from "react-native";
import * as FileSystem from 'expo-file-system';
-import { Audio } from "expo-av";
+import { AVPlaybackStatus, AVPlaybackStatusSuccess, Audio } from "expo-av";
interface MainProps {
route: {
@@ -19,30 +19,14 @@ const Main: React.FC = ({ route }) => {
const [audioQueue, setAudioQueue] = useState([]);
const [sound, setSound] = useState();
const audioDir = FileSystem.documentDirectory + '01/audio/';
- const Buffer = require('buffer').Buffer;
- const toBuffer = async (blob: Blob) => {
+ const constructTempFilePath = async (buffer: string) => {
+ await dirExists();
- const uri = await toDataURI(blob);
- const base64 = uri.replace(/^.*,/g, "");
- return Buffer.from(base64, "base64");
- };
-
- const toDataURI = (blob: Blob) =>
- new Promise((resolve) => {
- const reader = new FileReader();
- reader.readAsDataURL(blob);
- reader.onloadend = () => {
- const uri = reader.result?.toString();
- resolve(uri);
- };
- });
-
- const constructTempFilePath = async (buffer: Buffer) => {
const tempFilePath = `${audioDir}${Date.now()}.wav`;
await FileSystem.writeAsStringAsync(
tempFilePath,
- buffer.toString(),
+ buffer,
{
encoding: FileSystem.EncodingType.Base64,
}
@@ -66,6 +50,12 @@ const Main: React.FC = ({ route }) => {
const playNextAudio = async () => {
console.log("in playNextAudio audioQueue is", audioQueue.length);
+ if (sound != null){
+ console.log('Unloading Sound');
+ await sound.unloadAsync();
+ setSound(null);
+ }
+
if (audioQueue.length > 0) {
const uri = audioQueue.shift() as string;
console.log("load audio from", uri);
@@ -76,6 +66,9 @@ const Main: React.FC = ({ route }) => {
console.log("playing audio from", uri);
await sound?.playAsync();
+
+ sound.setOnPlaybackStatusUpdate(_onPlayBackStatusUpdate);
+
} catch (error){
console.log("Error playing audio", error);
playNextAudio();
@@ -84,16 +77,17 @@ const Main: React.FC = ({ route }) => {
}
};
- useEffect(() => {
- return sound
- ? () => {
- console.log('Unloading Sound');
- sound.unloadAsync();
- setSound(null);
- playNextAudio();
- }
- : undefined;
- }, [sound]);
+ const isAVPlaybackStatusSuccess = (
+ status: AVPlaybackStatus
+ ): status is AVPlaybackStatusSuccess => {
+ return (status as AVPlaybackStatusSuccess).isLoaded !== undefined;
+ };
+
+ const _onPlayBackStatusUpdate = (status: AVPlaybackStatus) => {
+ if (isAVPlaybackStatusSuccess(status) && status.didJustFinish){
+ playNextAudio();
+ }
+ }
useEffect(() => {
console.log("audioQueue has been updated:", audioQueue.length);
@@ -115,15 +109,16 @@ const Main: React.FC = ({ route }) => {
};
websocket.onmessage = async (e) => {
- console.log("Received message from WebSocket", e.data);
- const blob = await e.data;
- const buffer = await toBuffer(blob);
+ const message = JSON.parse(e.data);
+ console.log(message.content);
+
+ const buffer = await message.content;
const filePath = await constructTempFilePath(buffer);
setAudioQueue((prevQueue) => [...prevQueue, filePath]);
console.log("audio file written to", filePath);
- if (e.data.format === "bytes.raw" && e.data.end && audioQueue.length > 1) {
+ if (message.format === "bytes.raw" && message.end && audioQueue.length >= 1) {
console.log("calling playNextAudio");
playNextAudio();
}
diff --git a/software/source/server/server.py b/software/source/server/server.py
index dd71b06..a347026 100644
--- a/software/source/server/server.py
+++ b/software/source/server/server.py
@@ -393,8 +393,10 @@ def stream_tts(sentence):
with open(audio_file, "rb") as f:
audio_bytes = f.read()
- desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop')
- desktop_audio_file = os.path.join(desktop_path, os.path.basename(audio_file))
+ desktop_path = os.path.join(os.path.expanduser("~"), "Desktop")
+ desktop_audio_file = os.path.join(
+ desktop_path, f"{datetime.datetime.now()}" + os.path.basename(audio_file)
+ )
shutil.copy(audio_file, desktop_audio_file)
print(f"Audio file saved to Desktop: {desktop_audio_file}")
# storage_client = storage.Client(project="react-native-421323")
@@ -409,15 +411,23 @@ def stream_tts(sentence):
# f"Audio file {audio_file} uploaded to {datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav"
# )
- os.remove(audio_file)
-
file_type = "audio/wav"
# Read the entire WAV file
with open(audio_file, "rb") as f:
audio_bytes = f.read()
+ os.remove(audio_file)
+
# Stream the audio as a single message
- yield {"role": "assistant", "type": "audio", "format": file_type, "content": base64.b64encode(audio_bytes).decode('utf-8'), "start": True, "end": True}
+ yield {
+ "role": "assistant",
+ "type": "audio",
+ "format": file_type,
+ "content": base64.b64encode(audio_bytes).decode("utf-8"),
+ "start": True,
+ "end": True,
+ }
+
from uvicorn import Config, Server
import os
From d80bdbafe4918654c2451c9fbfcb3539e68e43db Mon Sep 17 00:00:00 2001
From: Ben Xu
Date: Thu, 25 Apr 2024 06:15:15 -0400
Subject: [PATCH 18/39] reset accumulator on server to process multiple
requests
---
.../ios/react-native/package-lock.json | 86 ++++++++++++++++-
.../clients/ios/react-native/package.json | 4 +-
.../ios/react-native/src/screens/Main.tsx | 96 ++++++++++---------
software/source/server/server.py | 5 +-
4 files changed, 142 insertions(+), 49 deletions(-)
diff --git a/software/source/clients/ios/react-native/package-lock.json b/software/source/clients/ios/react-native/package-lock.json
index 97b3864..40bb500 100644
--- a/software/source/clients/ios/react-native/package-lock.json
+++ b/software/source/clients/ios/react-native/package-lock.json
@@ -18,8 +18,10 @@
"react": "18.2.0",
"react-native": "0.73.4",
"react-native-base64": "^0.2.1",
+ "react-native-polyfill-globals": "^3.1.0",
"react-native-safe-area-context": "4.8.2",
- "react-native-screens": "~3.29.0"
+ "react-native-screens": "~3.29.0",
+ "text-encoding": "^0.7.0"
},
"devDependencies": {
"@babel/core": "^7.20.0",
@@ -6492,6 +6494,12 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
},
+ "node_modules/base-64": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/base-64/-/base-64-1.0.0.tgz",
+ "integrity": "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==",
+ "peer": true
+ },
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -7852,6 +7860,12 @@
"resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-1.11.1.tgz",
"integrity": "sha512-ddQEtCOgYHTLlFUe/yH67dDBIoct5VIULthyT3LRJbEwdpzAgueKsX2FYK02ldh440V87PWKCamh7R9evk1rrg=="
},
+ "node_modules/fast-base64-decode": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/fast-base64-decode/-/fast-base64-decode-1.0.0.tgz",
+ "integrity": "sha512-qwaScUgUGBYeDNRnbc/KyllVU88Jk1pRHPStuF/lO7B0/RTRLj7U0lkdTAutlBblY08rwZDff6tNU9cjv6j//Q==",
+ "peer": true
+ },
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -10782,6 +10796,15 @@
"os-tmpdir": "^1.0.0"
}
},
+ "node_modules/p-defer": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz",
+ "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==",
+ "peer": true,
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
@@ -11505,6 +11528,40 @@
"resolved": "https://registry.npmjs.org/react-native-base64/-/react-native-base64-0.2.1.tgz",
"integrity": "sha512-eHgt/MA8y5ZF0aHfZ1aTPcIkDWxza9AaEk4GcpIX+ZYfZ04RcaNahO+527KR7J44/mD3efYfM23O2C1N44ByWA=="
},
+ "node_modules/react-native-fetch-api": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/react-native-fetch-api/-/react-native-fetch-api-3.0.0.tgz",
+ "integrity": "sha512-g2rtqPjdroaboDKTsJCTlcmtw54E25OjyaunUP0anOZn4Fuo2IKs8BVfe02zVggA/UysbmfSnRJIqtNkAgggNA==",
+ "peer": true,
+ "dependencies": {
+ "p-defer": "^3.0.0"
+ }
+ },
+ "node_modules/react-native-get-random-values": {
+ "version": "1.11.0",
+ "resolved": "https://registry.npmjs.org/react-native-get-random-values/-/react-native-get-random-values-1.11.0.tgz",
+ "integrity": "sha512-4BTbDbRmS7iPdhYLRcz3PGFIpFJBwNZg9g42iwa2P6FOv9vZj/xJc678RZXnLNZzd0qd7Q3CCF6Yd+CU2eoXKQ==",
+ "peer": true,
+ "dependencies": {
+ "fast-base64-decode": "^1.0.0"
+ },
+ "peerDependencies": {
+ "react-native": ">=0.56"
+ }
+ },
+ "node_modules/react-native-polyfill-globals": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/react-native-polyfill-globals/-/react-native-polyfill-globals-3.1.0.tgz",
+ "integrity": "sha512-6ACmV1SjXvZP2LN6J2yK58yNACKddcvoiKLrSQdISx32IdYStfdmGXrbAfpd+TANrTlIaZ2SLoFXohNwhnqm/w==",
+ "peerDependencies": {
+ "base-64": "*",
+ "react-native-fetch-api": "*",
+ "react-native-get-random-values": "*",
+ "react-native-url-polyfill": "*",
+ "text-encoding": "*",
+ "web-streams-polyfill": "*"
+ }
+ },
"node_modules/react-native-safe-area-context": {
"version": "4.8.2",
"resolved": "https://registry.npmjs.org/react-native-safe-area-context/-/react-native-safe-area-context-4.8.2.tgz",
@@ -11527,6 +11584,18 @@
"react-native": "*"
}
},
+ "node_modules/react-native-url-polyfill": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/react-native-url-polyfill/-/react-native-url-polyfill-2.0.0.tgz",
+ "integrity": "sha512-My330Do7/DvKnEvwQc0WdcBnFPploYKp9CYlefDXzIdEaA+PAhDYllkvGeEroEzvc4Kzzj2O4yVdz8v6fjRvhA==",
+ "peer": true,
+ "dependencies": {
+ "whatwg-url-without-unicode": "8.0.0-3"
+ },
+ "peerDependencies": {
+ "react-native": "*"
+ }
+ },
"node_modules/react-native/node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
@@ -12589,6 +12658,12 @@
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
},
+ "node_modules/text-encoding": {
+ "version": "0.7.0",
+ "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.7.0.tgz",
+ "integrity": "sha512-oJQ3f1hrOnbRLOcwKz0Liq2IcrvDeZRHXhd9RgLrsT+DjWY/nty1Hi7v3dtkaEYbPYe0mUoOfzRrMwfXXwgPUA==",
+ "deprecated": "no longer maintained"
+ },
"node_modules/text-table": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@@ -12949,6 +13024,15 @@
"defaults": "^1.0.3"
}
},
+ "node_modules/web-streams-polyfill": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0.tgz",
+ "integrity": "sha512-0zJXHRAYEjM2tUfZ2DiSOHAa2aw1tisnnhU3ufD57R8iefL+DcdJyRBRyJpG+NUimDgbTI/lH+gAE1PAvV3Cgw==",
+ "peer": true,
+ "engines": {
+ "node": ">= 8"
+ }
+ },
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
diff --git a/software/source/clients/ios/react-native/package.json b/software/source/clients/ios/react-native/package.json
index 86faf84..6c2649a 100644
--- a/software/source/clients/ios/react-native/package.json
+++ b/software/source/clients/ios/react-native/package.json
@@ -20,8 +20,10 @@
"react": "18.2.0",
"react-native": "0.73.4",
"react-native-base64": "^0.2.1",
+ "react-native-polyfill-globals": "^3.1.0",
"react-native-safe-area-context": "4.8.2",
- "react-native-screens": "~3.29.0"
+ "react-native-screens": "~3.29.0",
+ "text-encoding": "^0.7.0"
},
"devDependencies": {
"@babel/core": "^7.20.0",
diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx
index a9a5ed3..28d3e75 100644
--- a/software/source/clients/ios/react-native/src/screens/Main.tsx
+++ b/software/source/clients/ios/react-native/src/screens/Main.tsx
@@ -1,7 +1,8 @@
-import React, { useState, useEffect } from "react";
+import React, { useState, useEffect, useCallback } from "react";
import { View, Text, TouchableOpacity, StyleSheet } from "react-native";
import * as FileSystem from 'expo-file-system';
import { AVPlaybackStatus, AVPlaybackStatusSuccess, Audio } from "expo-av";
+import { polyfill as polyfillEncoding } from 'react-native-polyfill-globals/src/encoding';
interface MainProps {
route: {
@@ -19,18 +20,23 @@ const Main: React.FC = ({ route }) => {
const [audioQueue, setAudioQueue] = useState([]);
const [sound, setSound] = useState();
const audioDir = FileSystem.documentDirectory + '01/audio/';
+ const [permissionResponse, requestPermission] = Audio.usePermissions();
+ polyfillEncoding();
+ const reader = new FileReader();
const constructTempFilePath = async (buffer: string) => {
await dirExists();
-
const tempFilePath = `${audioDir}${Date.now()}.wav`;
- await FileSystem.writeAsStringAsync(
- tempFilePath,
- buffer,
- {
- encoding: FileSystem.EncodingType.Base64,
- }
- );
+
+
+ await FileSystem.writeAsStringAsync(
+ tempFilePath,
+ buffer,
+ {
+ encoding: FileSystem.EncodingType.Base64,
+ }
+ );
+
return tempFilePath;
};
@@ -111,9 +117,9 @@ const Main: React.FC = ({ route }) => {
websocket.onmessage = async (e) => {
const message = JSON.parse(e.data);
- console.log(message.content);
+ console.log(message.content.slice(0, 50));
- const buffer = await message.content;
+ const buffer = await message.content as string;
const filePath = await constructTempFilePath(buffer);
setAudioQueue((prevQueue) => [...prevQueue, filePath]);
console.log("audio file written to", filePath);
@@ -122,26 +128,6 @@ const Main: React.FC = ({ route }) => {
console.log("calling playNextAudio");
playNextAudio();
}
-
- /**
- const message = JSON.parse(e.data);
-
- if (message.content) {
- const parsedMessage = message.content.replace(/^b'|['"]|['"]$/g, "");
- console.log("parsedMessage", parsedMessage.slice(0, 30));
-
- const filePath = await constructFilePath(parsedMessage);
- setAudioQueue((prevQueue) => [...prevQueue, filePath]);
- console.log("audio file written to", filePath);
- }
-
- if (message.format === "bytes.raw" && message.end && audioQueue.length > 1) {
- console.log("calling playNextAudio");
- playNextAudio();
- }
-
- */
-
};
websocket.onerror = (error) => {
@@ -167,56 +153,76 @@ const Main: React.FC = ({ route }) => {
};
}, [scannedData]);
- const startRecording = async () => {
+ const startRecording = useCallback(async () => {
if (recording) {
console.log("A recording is already in progress.");
return;
}
try {
- console.log("Requesting permissions..");
- await Audio.requestPermissionsAsync();
+ if (permissionResponse !== null && permissionResponse.status !== `granted`) {
+ console.log("Requesting permission..");
+ await requestPermission();
+ }
+
await Audio.setAudioModeAsync({
allowsRecordingIOS: true,
playsInSilentModeIOS: true,
});
+
console.log("Starting recording..");
- const { recording: newRecording } = await Audio.Recording.createAsync(
- Audio.RecordingOptionsPresets.HIGH_QUALITY
- );
+ const newRecording = new Audio.Recording();
+ await newRecording.prepareToRecordAsync(Audio.RecordingOptionsPresets.HIGH_QUALITY);
+ await newRecording.startAsync();
+
setRecording(newRecording);
- console.log("Recording started");
} catch (err) {
console.error("Failed to start recording", err);
}
- };
+ }, []);
- const stopRecording = async () => {
+ const stopRecording = useCallback(async () => {
console.log("Stopping recording..");
- setRecording(null);
+
if (recording) {
await recording.stopAndUnloadAsync();
await Audio.setAudioModeAsync({
allowsRecordingIOS: false,
});
const uri = recording.getURI();
- console.log("Recording stopped and stored at", uri);
+ console.log("recording uri at ", uri);
+ setRecording(null);
+
+ // sanity check play the audio recording locally
+ // recording is working fine; is the server caching the audio file somewhere?
+ /**
+ if (uri) {
+ const { sound } = await Audio.Sound.createAsync({ uri });
+ sound.playAsync();
+ console.log("playing audio recording from", uri);
+ }
+ */
+
if (ws && uri) {
const response = await fetch(uri);
+ console.log("fetched audio file", response);
const blob = await response.blob();
- const reader = new FileReader();
+
reader.readAsArrayBuffer(blob);
reader.onloadend = () => {
const audioBytes = reader.result;
if (audioBytes) {
ws.send(audioBytes);
- console.log("sent audio bytes to WebSocket");
+ const audioArray = new Uint8Array(audioBytes as ArrayBuffer);
+ const decoder = new TextDecoder("utf-8");
+ console.log("sent audio bytes to WebSocket", decoder.decode(audioArray).slice(0, 50));
}
};
}
+
}
- };
+ }, [recording]);
return (
diff --git a/software/source/server/server.py b/software/source/server/server.py
index a347026..a1a7ef2 100644
--- a/software/source/server/server.py
+++ b/software/source/server/server.py
@@ -39,8 +39,6 @@ print("")
setup_logging()
-accumulator = Accumulator()
-
app = FastAPI()
app_dir = user_data_dir("01")
@@ -229,6 +227,8 @@ async def send_messages(websocket: WebSocket):
async def listener():
while True:
try:
+ accumulator = Accumulator()
+
while True:
if not from_user.empty():
chunk = await from_user.get()
@@ -258,6 +258,7 @@ async def listener():
# Convert bytes to audio file
# Format will be bytes.wav or bytes.opus
mime_type = "audio/" + message["format"].split(".")[1]
+ print("input audio file content", message["content"][:100])
audio_file_path = bytes_to_wav(message["content"], mime_type)
print("Audio file path:", audio_file_path)
From cd22f223000491216e77dd07b28643c426477ad8 Mon Sep 17 00:00:00 2001
From: Ben Xu
Date: Thu, 25 Apr 2024 08:24:02 -0400
Subject: [PATCH 19/39] add onPlayBackStatusUpdate
---
.../ios/react-native/package-lock.json | 46 ++++++++++--
.../clients/ios/react-native/package.json | 3 +-
.../ios/react-native/src/screens/Main.tsx | 74 +++++++++++++------
3 files changed, 93 insertions(+), 30 deletions(-)
diff --git a/software/source/clients/ios/react-native/package-lock.json b/software/source/clients/ios/react-native/package-lock.json
index 40bb500..86dd5f3 100644
--- a/software/source/clients/ios/react-native/package-lock.json
+++ b/software/source/clients/ios/react-native/package-lock.json
@@ -21,7 +21,8 @@
"react-native-polyfill-globals": "^3.1.0",
"react-native-safe-area-context": "4.8.2",
"react-native-screens": "~3.29.0",
- "text-encoding": "^0.7.0"
+ "text-encoding": "^0.7.0",
+ "zustand": "^4.5.2"
},
"devDependencies": {
"@babel/core": "^7.20.0",
@@ -6093,13 +6094,13 @@
"version": "15.7.11",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz",
"integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==",
- "dev": true
+ "devOptional": true
},
"node_modules/@types/react": {
"version": "18.2.63",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.63.tgz",
"integrity": "sha512-ppaqODhs15PYL2nGUOaOu2RSCCB4Difu4UFrP4I3NHLloXC/ESQzQMi9nvjfT1+rudd0d2L3fQPJxRSey+rGlQ==",
- "dev": true,
+ "devOptional": true,
"dependencies": {
"@types/prop-types": "*",
"@types/scheduler": "*",
@@ -6116,7 +6117,7 @@
"version": "0.16.8",
"resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz",
"integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==",
- "dev": true
+ "devOptional": true
},
"node_modules/@types/stack-utils": {
"version": "2.0.3",
@@ -7243,7 +7244,7 @@
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
- "dev": true
+ "devOptional": true
},
"node_modules/dag-map": {
"version": "1.0.2",
@@ -12956,6 +12957,14 @@
"react": ">=16.8"
}
},
+ "node_modules/use-sync-external-store": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz",
+ "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==",
+ "peerDependencies": {
+ "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+ }
+ },
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -13301,6 +13310,33 @@
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
+ },
+ "node_modules/zustand": {
+ "version": "4.5.2",
+ "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.2.tgz",
+ "integrity": "sha512-2cN1tPkDVkwCy5ickKrI7vijSjPksFRfqS6237NzT0vqSsztTNnQdHw9mmN7uBdk3gceVXU0a+21jFzFzAc9+g==",
+ "dependencies": {
+ "use-sync-external-store": "1.2.0"
+ },
+ "engines": {
+ "node": ">=12.7.0"
+ },
+ "peerDependencies": {
+ "@types/react": ">=16.8",
+ "immer": ">=9.0.6",
+ "react": ">=16.8"
+ },
+ "peerDependenciesMeta": {
+ "@types/react": {
+ "optional": true
+ },
+ "immer": {
+ "optional": true
+ },
+ "react": {
+ "optional": true
+ }
+ }
}
}
}
diff --git a/software/source/clients/ios/react-native/package.json b/software/source/clients/ios/react-native/package.json
index 6c2649a..58772a2 100644
--- a/software/source/clients/ios/react-native/package.json
+++ b/software/source/clients/ios/react-native/package.json
@@ -23,7 +23,8 @@
"react-native-polyfill-globals": "^3.1.0",
"react-native-safe-area-context": "4.8.2",
"react-native-screens": "~3.29.0",
- "text-encoding": "^0.7.0"
+ "text-encoding": "^0.7.0",
+ "zustand": "^4.5.2"
},
"devDependencies": {
"@babel/core": "^7.20.0",
diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx
index 28d3e75..0e0ad62 100644
--- a/software/source/clients/ios/react-native/src/screens/Main.tsx
+++ b/software/source/clients/ios/react-native/src/screens/Main.tsx
@@ -3,6 +3,7 @@ import { View, Text, TouchableOpacity, StyleSheet } from "react-native";
import * as FileSystem from 'expo-file-system';
import { AVPlaybackStatus, AVPlaybackStatusSuccess, Audio } from "expo-av";
import { polyfill as polyfillEncoding } from 'react-native-polyfill-globals/src/encoding';
+import { create } from 'zustand';
interface MainProps {
route: {
@@ -12,23 +13,43 @@ interface MainProps {
};
}
+interface AudioQueueState {
+ audioQueue: string[]; // Define the audio queue type
+ addToQueue: (uri: string) => void; // Function to set audio queue
+}
+
+const useAudioQueueStore = create((set) => ({
+ audioQueue: [], // initial state
+ addToQueue: (uri) => set((state) => ({ audioQueue: [...state.audioQueue, uri] })), // action to set audio queue
+}));
+
+interface SoundState {
+ sound: Audio.Sound | null; // Define the sound type
+ setSound: (newSound: Audio.Sound | null) => void; // Function to set sound
+}
+
+const useSoundStore = create((set) => ({
+ sound: null, // initial state
+ setSound: (newSound) => set({ sound: newSound }), // action to set sound
+}));
+
const Main: React.FC = ({ route }) => {
const { scannedData } = route.params;
const [connectionStatus, setConnectionStatus] = useState("Connecting...");
const [ws, setWs] = useState(null);
const [recording, setRecording] = useState(null);
- const [audioQueue, setAudioQueue] = useState([]);
- const [sound, setSound] = useState();
+ const addToQueue = useAudioQueueStore((state) => state.addToQueue);
+ const audioQueue = useAudioQueueStore((state) => state.audioQueue);
+ const setSound = useSoundStore((state) => state.setSound);
+ const sound = useSoundStore((state) => state.sound);
const audioDir = FileSystem.documentDirectory + '01/audio/';
const [permissionResponse, requestPermission] = Audio.usePermissions();
polyfillEncoding();
- const reader = new FileReader();
const constructTempFilePath = async (buffer: string) => {
await dirExists();
const tempFilePath = `${audioDir}${Date.now()}.wav`;
-
await FileSystem.writeAsStringAsync(
tempFilePath,
buffer,
@@ -37,7 +58,6 @@ const Main: React.FC = ({ route }) => {
}
);
-
return tempFilePath;
};
@@ -54,15 +74,10 @@ const Main: React.FC = ({ route }) => {
}
const playNextAudio = async () => {
- console.log("in playNextAudio audioQueue is", audioQueue.length);
+ console.log(`in playNextAudio audioQueue is ${audioQueue.length} and sound is ${sound}`);
- if (sound != null){
- console.log('Unloading Sound');
- await sound.unloadAsync();
- setSound(null);
- }
- if (audioQueue.length > 0) {
+ if (audioQueue.length > 0 && sound == null) {
const uri = audioQueue.shift() as string;
console.log("load audio from", uri);
@@ -80,21 +95,32 @@ const Main: React.FC = ({ route }) => {
playNextAudio();
}
+ } else {
+ console.log("audioQueue is empty or sound is not null");
+ return;
}
};
+ const _onPlayBackStatusUpdate = async (status: AVPlaybackStatus) => {
+ if (isAVPlaybackStatusSuccess(status) && status.didJustFinish === true){
+ console.log("on playback status update sound is ", sound);
+ if (sound != null){
+ console.log('Unloading Sound');
+ await sound.unloadAsync();
+ }
+ setSound(null);
+ console.log("audio has finished playing, playing next audio");
+ console.log(audioQueue);
+ playNextAudio();
+ }
+ }
+
const isAVPlaybackStatusSuccess = (
status: AVPlaybackStatus
): status is AVPlaybackStatusSuccess => {
return (status as AVPlaybackStatusSuccess).isLoaded !== undefined;
};
- const _onPlayBackStatusUpdate = (status: AVPlaybackStatus) => {
- if (isAVPlaybackStatusSuccess(status) && status.didJustFinish){
- playNextAudio();
- }
- }
-
useEffect(() => {
console.log("audioQueue has been updated:", audioQueue.length);
if (audioQueue.length == 1) {
@@ -102,6 +128,10 @@ const Main: React.FC = ({ route }) => {
}
}, [audioQueue]);
+ useEffect(() => {
+ console.log("sound has been updated:", sound);
+ }, [sound]);
+
useEffect(() => {
let websocket: WebSocket;
try {
@@ -121,13 +151,8 @@ const Main: React.FC = ({ route }) => {
const buffer = await message.content as string;
const filePath = await constructTempFilePath(buffer);
- setAudioQueue((prevQueue) => [...prevQueue, filePath]);
+ addToQueue(filePath);
console.log("audio file written to", filePath);
-
- if (message.format === "bytes.raw" && message.end && audioQueue.length >= 1) {
- console.log("calling playNextAudio");
- playNextAudio();
- }
};
websocket.onerror = (error) => {
@@ -209,6 +234,7 @@ const Main: React.FC = ({ route }) => {
console.log("fetched audio file", response);
const blob = await response.blob();
+ const reader = new FileReader();
reader.readAsArrayBuffer(blob);
reader.onloadend = () => {
const audioBytes = reader.result;
From 3efe9aca8c43b9696d8f79b399b7148a9dbe9aed Mon Sep 17 00:00:00 2001
From: Ikko Eltociear Ashimine
Date: Fri, 26 Apr 2024 00:40:39 +0900
Subject: [PATCH 20/39] Update server.py
recieved -> received
---
software/source/server/server.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/software/source/server/server.py b/software/source/server/server.py
index c4dd036..d5dbb39 100644
--- a/software/source/server/server.py
+++ b/software/source/server/server.py
@@ -343,7 +343,7 @@ async def listener():
json.dump(interpreter.messages, file, indent=4)
# TODO: is triggering seemingly randomly
- # logger.info("New user message recieved. Breaking.")
+ # logger.info("New user message received. Breaking.")
# break
# Also check if there's any new computer messages
@@ -351,7 +351,7 @@ async def listener():
with open(conversation_history_path, "w") as file:
json.dump(interpreter.messages, file, indent=4)
- logger.info("New computer message recieved. Breaking.")
+ logger.info("New computer message received. Breaking.")
break
except:
traceback.print_exc()
From 4647b24ccc0180548ee1804f192424859cab42db Mon Sep 17 00:00:00 2001
From: Ty Fiero
Date: Thu, 25 Apr 2024 12:05:12 -0700
Subject: [PATCH 21/39] It works!!
---
.../ios/react-native/package-lock.json | 9 +
.../clients/ios/react-native/package.json | 3 +-
.../react-native/src/screens/HomeScreen.tsx | 2 +-
.../ios/react-native/src/screens/Main.tsx | 252 ++++++++++++------
4 files changed, 183 insertions(+), 83 deletions(-)
diff --git a/software/source/clients/ios/react-native/package-lock.json b/software/source/clients/ios/react-native/package-lock.json
index 86dd5f3..e78cdd4 100644
--- a/software/source/clients/ios/react-native/package-lock.json
+++ b/software/source/clients/ios/react-native/package-lock.json
@@ -14,6 +14,7 @@
"expo-av": "~13.10.5",
"expo-barcode-scanner": "~12.9.3",
"expo-camera": "~14.0.5",
+ "expo-haptics": "~12.8.1",
"expo-status-bar": "~1.11.1",
"react": "18.2.0",
"react-native": "0.73.4",
@@ -7719,6 +7720,14 @@
"expo": "*"
}
},
+ "node_modules/expo-haptics": {
+ "version": "12.8.1",
+ "resolved": "https://registry.npmjs.org/expo-haptics/-/expo-haptics-12.8.1.tgz",
+ "integrity": "sha512-ntLsHkfle8K8w9MW8pZEw92ZN3sguaGUSSIxv30fPKNeQFu7Cq/h47Qv3tONv2MO3wU48N9FbKnant6XlfptpA==",
+ "peerDependencies": {
+ "expo": "*"
+ }
+ },
"node_modules/expo-image-loader": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/expo-image-loader/-/expo-image-loader-4.6.0.tgz",
diff --git a/software/source/clients/ios/react-native/package.json b/software/source/clients/ios/react-native/package.json
index 58772a2..6d5cbe2 100644
--- a/software/source/clients/ios/react-native/package.json
+++ b/software/source/clients/ios/react-native/package.json
@@ -24,7 +24,8 @@
"react-native-safe-area-context": "4.8.2",
"react-native-screens": "~3.29.0",
"text-encoding": "^0.7.0",
- "zustand": "^4.5.2"
+ "zustand": "^4.5.2",
+ "expo-haptics": "~12.8.1"
},
"devDependencies": {
"@babel/core": "^7.20.0",
diff --git a/software/source/clients/ios/react-native/src/screens/HomeScreen.tsx b/software/source/clients/ios/react-native/src/screens/HomeScreen.tsx
index cb0644b..27ddd4f 100644
--- a/software/source/clients/ios/react-native/src/screens/HomeScreen.tsx
+++ b/software/source/clients/ios/react-native/src/screens/HomeScreen.tsx
@@ -7,7 +7,7 @@ const HomeScreen = () => {
return (
-
+ {/* */}
navigation.navigate("Camera")}
diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx
index 0e0ad62..5c360ab 100644
--- a/software/source/clients/ios/react-native/src/screens/Main.tsx
+++ b/software/source/clients/ios/react-native/src/screens/Main.tsx
@@ -1,9 +1,12 @@
-import React, { useState, useEffect, useCallback } from "react";
+import React, { useState, useEffect, useCallback, useRef } from "react";
import { View, Text, TouchableOpacity, StyleSheet } from "react-native";
-import * as FileSystem from 'expo-file-system';
+import * as FileSystem from "expo-file-system";
import { AVPlaybackStatus, AVPlaybackStatusSuccess, Audio } from "expo-av";
-import { polyfill as polyfillEncoding } from 'react-native-polyfill-globals/src/encoding';
-import { create } from 'zustand';
+import { polyfill as polyfillEncoding } from "react-native-polyfill-globals/src/encoding";
+import { create } from "zustand";
+import useStore from "../lib/state";
+import { Animated } from "react-native";
+import * as Haptics from "expo-haptics";
interface MainProps {
route: {
@@ -20,7 +23,8 @@ interface AudioQueueState {
const useAudioQueueStore = create((set) => ({
audioQueue: [], // initial state
- addToQueue: (uri) => set((state) => ({ audioQueue: [...state.audioQueue, uri] })), // action to set audio queue
+ addToQueue: (uri) =>
+ set((state) => ({ audioQueue: [...state.audioQueue, uri] })), // action to set audio queue
}));
interface SoundState {
@@ -35,85 +39,105 @@ const useSoundStore = create((set) => ({
const Main: React.FC = ({ route }) => {
const { scannedData } = route.params;
- const [connectionStatus, setConnectionStatus] = useState("Connecting...");
+ const [connectionStatus, setConnectionStatus] =
+ useState("Connecting...");
const [ws, setWs] = useState(null);
+ const [isPressed, setIsPressed] = useState(false);
const [recording, setRecording] = useState(null);
const addToQueue = useAudioQueueStore((state) => state.addToQueue);
const audioQueue = useAudioQueueStore((state) => state.audioQueue);
const setSound = useSoundStore((state) => state.setSound);
const sound = useSoundStore((state) => state.sound);
- const audioDir = FileSystem.documentDirectory + '01/audio/';
+ const [soundUriMap, setSoundUriMap] = useState