From d80bdbafe4918654c2451c9fbfcb3539e68e43db Mon Sep 17 00:00:00 2001 From: Ben Xu Date: Thu, 25 Apr 2024 06:15:15 -0400 Subject: [PATCH] reset accumulator on server to process multiple requests --- .../ios/react-native/package-lock.json | 86 ++++++++++++++++- .../clients/ios/react-native/package.json | 4 +- .../ios/react-native/src/screens/Main.tsx | 96 ++++++++++--------- software/source/server/server.py | 5 +- 4 files changed, 142 insertions(+), 49 deletions(-) diff --git a/software/source/clients/ios/react-native/package-lock.json b/software/source/clients/ios/react-native/package-lock.json index 97b3864..40bb500 100644 --- a/software/source/clients/ios/react-native/package-lock.json +++ b/software/source/clients/ios/react-native/package-lock.json @@ -18,8 +18,10 @@ "react": "18.2.0", "react-native": "0.73.4", "react-native-base64": "^0.2.1", + "react-native-polyfill-globals": "^3.1.0", "react-native-safe-area-context": "4.8.2", - "react-native-screens": "~3.29.0" + "react-native-screens": "~3.29.0", + "text-encoding": "^0.7.0" }, "devDependencies": { "@babel/core": "^7.20.0", @@ -6492,6 +6494,12 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base-64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-1.0.0.tgz", + "integrity": "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==", + "peer": true + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -7852,6 +7860,12 @@ "resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-1.11.1.tgz", "integrity": "sha512-ddQEtCOgYHTLlFUe/yH67dDBIoct5VIULthyT3LRJbEwdpzAgueKsX2FYK02ldh440V87PWKCamh7R9evk1rrg==" }, + "node_modules/fast-base64-decode": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-base64-decode/-/fast-base64-decode-1.0.0.tgz", + "integrity": "sha512-qwaScUgUGBYeDNRnbc/KyllVU88Jk1pRHPStuF/lO7B0/RTRLj7U0lkdTAutlBblY08rwZDff6tNU9cjv6j//Q==", + "peer": true + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -10782,6 +10796,15 @@ "os-tmpdir": "^1.0.0" } }, + "node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -11505,6 +11528,40 @@ "resolved": "https://registry.npmjs.org/react-native-base64/-/react-native-base64-0.2.1.tgz", "integrity": "sha512-eHgt/MA8y5ZF0aHfZ1aTPcIkDWxza9AaEk4GcpIX+ZYfZ04RcaNahO+527KR7J44/mD3efYfM23O2C1N44ByWA==" }, + "node_modules/react-native-fetch-api": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/react-native-fetch-api/-/react-native-fetch-api-3.0.0.tgz", + "integrity": "sha512-g2rtqPjdroaboDKTsJCTlcmtw54E25OjyaunUP0anOZn4Fuo2IKs8BVfe02zVggA/UysbmfSnRJIqtNkAgggNA==", + "peer": true, + "dependencies": { + "p-defer": "^3.0.0" + } + }, + "node_modules/react-native-get-random-values": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/react-native-get-random-values/-/react-native-get-random-values-1.11.0.tgz", + "integrity": "sha512-4BTbDbRmS7iPdhYLRcz3PGFIpFJBwNZg9g42iwa2P6FOv9vZj/xJc678RZXnLNZzd0qd7Q3CCF6Yd+CU2eoXKQ==", + "peer": true, + "dependencies": { + "fast-base64-decode": "^1.0.0" + }, + "peerDependencies": { + "react-native": ">=0.56" + } + }, + "node_modules/react-native-polyfill-globals": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/react-native-polyfill-globals/-/react-native-polyfill-globals-3.1.0.tgz", + "integrity": "sha512-6ACmV1SjXvZP2LN6J2yK58yNACKddcvoiKLrSQdISx32IdYStfdmGXrbAfpd+TANrTlIaZ2SLoFXohNwhnqm/w==", + "peerDependencies": { + "base-64": "*", + "react-native-fetch-api": "*", + "react-native-get-random-values": "*", + "react-native-url-polyfill": "*", + "text-encoding": "*", + "web-streams-polyfill": "*" + } + }, "node_modules/react-native-safe-area-context": { "version": "4.8.2", "resolved": "https://registry.npmjs.org/react-native-safe-area-context/-/react-native-safe-area-context-4.8.2.tgz", @@ -11527,6 +11584,18 @@ "react-native": "*" } }, + "node_modules/react-native-url-polyfill": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/react-native-url-polyfill/-/react-native-url-polyfill-2.0.0.tgz", + "integrity": "sha512-My330Do7/DvKnEvwQc0WdcBnFPploYKp9CYlefDXzIdEaA+PAhDYllkvGeEroEzvc4Kzzj2O4yVdz8v6fjRvhA==", + "peer": true, + "dependencies": { + "whatwg-url-without-unicode": "8.0.0-3" + }, + "peerDependencies": { + "react-native": "*" + } + }, "node_modules/react-native/node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -12589,6 +12658,12 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, + "node_modules/text-encoding": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.7.0.tgz", + "integrity": "sha512-oJQ3f1hrOnbRLOcwKz0Liq2IcrvDeZRHXhd9RgLrsT+DjWY/nty1Hi7v3dtkaEYbPYe0mUoOfzRrMwfXXwgPUA==", + "deprecated": "no longer maintained" + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -12949,6 +13024,15 @@ "defaults": "^1.0.3" } }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0.tgz", + "integrity": "sha512-0zJXHRAYEjM2tUfZ2DiSOHAa2aw1tisnnhU3ufD57R8iefL+DcdJyRBRyJpG+NUimDgbTI/lH+gAE1PAvV3Cgw==", + "peer": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/software/source/clients/ios/react-native/package.json b/software/source/clients/ios/react-native/package.json index 86faf84..6c2649a 100644 --- a/software/source/clients/ios/react-native/package.json +++ b/software/source/clients/ios/react-native/package.json @@ -20,8 +20,10 @@ "react": "18.2.0", "react-native": "0.73.4", "react-native-base64": "^0.2.1", + "react-native-polyfill-globals": "^3.1.0", "react-native-safe-area-context": "4.8.2", - "react-native-screens": "~3.29.0" + "react-native-screens": "~3.29.0", + "text-encoding": "^0.7.0" }, "devDependencies": { "@babel/core": "^7.20.0", diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx index a9a5ed3..28d3e75 100644 --- a/software/source/clients/ios/react-native/src/screens/Main.tsx +++ b/software/source/clients/ios/react-native/src/screens/Main.tsx @@ -1,7 +1,8 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useCallback } from "react"; import { View, Text, TouchableOpacity, StyleSheet } from "react-native"; import * as FileSystem from 'expo-file-system'; import { AVPlaybackStatus, AVPlaybackStatusSuccess, Audio } from "expo-av"; +import { polyfill as polyfillEncoding } from 'react-native-polyfill-globals/src/encoding'; interface MainProps { route: { @@ -19,18 +20,23 @@ const Main: React.FC = ({ route }) => { const [audioQueue, setAudioQueue] = useState([]); const [sound, setSound] = useState(); const audioDir = FileSystem.documentDirectory + '01/audio/'; + const [permissionResponse, requestPermission] = Audio.usePermissions(); + polyfillEncoding(); + const reader = new FileReader(); const constructTempFilePath = async (buffer: string) => { await dirExists(); - const tempFilePath = `${audioDir}${Date.now()}.wav`; - await FileSystem.writeAsStringAsync( - tempFilePath, - buffer, - { - encoding: FileSystem.EncodingType.Base64, - } - ); + + + await FileSystem.writeAsStringAsync( + tempFilePath, + buffer, + { + encoding: FileSystem.EncodingType.Base64, + } + ); + return tempFilePath; }; @@ -111,9 +117,9 @@ const Main: React.FC = ({ route }) => { websocket.onmessage = async (e) => { const message = JSON.parse(e.data); - console.log(message.content); + console.log(message.content.slice(0, 50)); - const buffer = await message.content; + const buffer = await message.content as string; const filePath = await constructTempFilePath(buffer); setAudioQueue((prevQueue) => [...prevQueue, filePath]); console.log("audio file written to", filePath); @@ -122,26 +128,6 @@ const Main: React.FC = ({ route }) => { console.log("calling playNextAudio"); playNextAudio(); } - - /** - const message = JSON.parse(e.data); - - if (message.content) { - const parsedMessage = message.content.replace(/^b'|['"]|['"]$/g, ""); - console.log("parsedMessage", parsedMessage.slice(0, 30)); - - const filePath = await constructFilePath(parsedMessage); - setAudioQueue((prevQueue) => [...prevQueue, filePath]); - console.log("audio file written to", filePath); - } - - if (message.format === "bytes.raw" && message.end && audioQueue.length > 1) { - console.log("calling playNextAudio"); - playNextAudio(); - } - - */ - }; websocket.onerror = (error) => { @@ -167,56 +153,76 @@ const Main: React.FC = ({ route }) => { }; }, [scannedData]); - const startRecording = async () => { + const startRecording = useCallback(async () => { if (recording) { console.log("A recording is already in progress."); return; } try { - console.log("Requesting permissions.."); - await Audio.requestPermissionsAsync(); + if (permissionResponse !== null && permissionResponse.status !== `granted`) { + console.log("Requesting permission.."); + await requestPermission(); + } + await Audio.setAudioModeAsync({ allowsRecordingIOS: true, playsInSilentModeIOS: true, }); + console.log("Starting recording.."); - const { recording: newRecording } = await Audio.Recording.createAsync( - Audio.RecordingOptionsPresets.HIGH_QUALITY - ); + const newRecording = new Audio.Recording(); + await newRecording.prepareToRecordAsync(Audio.RecordingOptionsPresets.HIGH_QUALITY); + await newRecording.startAsync(); + setRecording(newRecording); - console.log("Recording started"); } catch (err) { console.error("Failed to start recording", err); } - }; + }, []); - const stopRecording = async () => { + const stopRecording = useCallback(async () => { console.log("Stopping recording.."); - setRecording(null); + if (recording) { await recording.stopAndUnloadAsync(); await Audio.setAudioModeAsync({ allowsRecordingIOS: false, }); const uri = recording.getURI(); - console.log("Recording stopped and stored at", uri); + console.log("recording uri at ", uri); + setRecording(null); + + // sanity check play the audio recording locally + // recording is working fine; is the server caching the audio file somewhere? + /** + if (uri) { + const { sound } = await Audio.Sound.createAsync({ uri }); + sound.playAsync(); + console.log("playing audio recording from", uri); + } + */ + if (ws && uri) { const response = await fetch(uri); + console.log("fetched audio file", response); const blob = await response.blob(); - const reader = new FileReader(); + reader.readAsArrayBuffer(blob); reader.onloadend = () => { const audioBytes = reader.result; if (audioBytes) { ws.send(audioBytes); - console.log("sent audio bytes to WebSocket"); + const audioArray = new Uint8Array(audioBytes as ArrayBuffer); + const decoder = new TextDecoder("utf-8"); + console.log("sent audio bytes to WebSocket", decoder.decode(audioArray).slice(0, 50)); } }; } + } - }; + }, [recording]); return ( diff --git a/software/source/server/server.py b/software/source/server/server.py index a347026..a1a7ef2 100644 --- a/software/source/server/server.py +++ b/software/source/server/server.py @@ -39,8 +39,6 @@ print("") setup_logging() -accumulator = Accumulator() - app = FastAPI() app_dir = user_data_dir("01") @@ -229,6 +227,8 @@ async def send_messages(websocket: WebSocket): async def listener(): while True: try: + accumulator = Accumulator() + while True: if not from_user.empty(): chunk = await from_user.get() @@ -258,6 +258,7 @@ async def listener(): # Convert bytes to audio file # Format will be bytes.wav or bytes.opus mime_type = "audio/" + message["format"].split(".")[1] + print("input audio file content", message["content"][:100]) audio_file_path = bytes_to_wav(message["content"], mime_type) print("Audio file path:", audio_file_path)