From cd22f223000491216e77dd07b28643c426477ad8 Mon Sep 17 00:00:00 2001 From: Ben Xu Date: Thu, 25 Apr 2024 08:24:02 -0400 Subject: [PATCH] add onPlayBackStatusUpdate --- .../ios/react-native/package-lock.json | 46 ++++++++++-- .../clients/ios/react-native/package.json | 3 +- .../ios/react-native/src/screens/Main.tsx | 74 +++++++++++++------ 3 files changed, 93 insertions(+), 30 deletions(-) diff --git a/software/source/clients/ios/react-native/package-lock.json b/software/source/clients/ios/react-native/package-lock.json index 40bb500..86dd5f3 100644 --- a/software/source/clients/ios/react-native/package-lock.json +++ b/software/source/clients/ios/react-native/package-lock.json @@ -21,7 +21,8 @@ "react-native-polyfill-globals": "^3.1.0", "react-native-safe-area-context": "4.8.2", "react-native-screens": "~3.29.0", - "text-encoding": "^0.7.0" + "text-encoding": "^0.7.0", + "zustand": "^4.5.2" }, "devDependencies": { "@babel/core": "^7.20.0", @@ -6093,13 +6094,13 @@ "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==", - "dev": true + "devOptional": true }, "node_modules/@types/react": { "version": "18.2.63", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.63.tgz", "integrity": "sha512-ppaqODhs15PYL2nGUOaOu2RSCCB4Difu4UFrP4I3NHLloXC/ESQzQMi9nvjfT1+rudd0d2L3fQPJxRSey+rGlQ==", - "dev": true, + "devOptional": true, "dependencies": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -6116,7 +6117,7 @@ "version": "0.16.8", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz", "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==", - "dev": true + "devOptional": true }, "node_modules/@types/stack-utils": { "version": "2.0.3", @@ -7243,7 +7244,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", - "dev": true + "devOptional": true }, "node_modules/dag-map": { "version": "1.0.2", @@ -12956,6 +12957,14 @@ "react": ">=16.8" } }, + "node_modules/use-sync-external-store": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", + "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -13301,6 +13310,33 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zustand": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.2.tgz", + "integrity": "sha512-2cN1tPkDVkwCy5ickKrI7vijSjPksFRfqS6237NzT0vqSsztTNnQdHw9mmN7uBdk3gceVXU0a+21jFzFzAc9+g==", + "dependencies": { + "use-sync-external-store": "1.2.0" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0.6", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } } } } diff --git a/software/source/clients/ios/react-native/package.json b/software/source/clients/ios/react-native/package.json index 6c2649a..58772a2 100644 --- a/software/source/clients/ios/react-native/package.json +++ b/software/source/clients/ios/react-native/package.json @@ -23,7 +23,8 @@ "react-native-polyfill-globals": "^3.1.0", "react-native-safe-area-context": "4.8.2", "react-native-screens": "~3.29.0", - "text-encoding": "^0.7.0" + "text-encoding": "^0.7.0", + "zustand": "^4.5.2" }, "devDependencies": { "@babel/core": "^7.20.0", diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx index 28d3e75..0e0ad62 100644 --- a/software/source/clients/ios/react-native/src/screens/Main.tsx +++ b/software/source/clients/ios/react-native/src/screens/Main.tsx @@ -3,6 +3,7 @@ import { View, Text, TouchableOpacity, StyleSheet } from "react-native"; import * as FileSystem from 'expo-file-system'; import { AVPlaybackStatus, AVPlaybackStatusSuccess, Audio } from "expo-av"; import { polyfill as polyfillEncoding } from 'react-native-polyfill-globals/src/encoding'; +import { create } from 'zustand'; interface MainProps { route: { @@ -12,23 +13,43 @@ interface MainProps { }; } +interface AudioQueueState { + audioQueue: string[]; // Define the audio queue type + addToQueue: (uri: string) => void; // Function to set audio queue +} + +const useAudioQueueStore = create((set) => ({ + audioQueue: [], // initial state + addToQueue: (uri) => set((state) => ({ audioQueue: [...state.audioQueue, uri] })), // action to set audio queue +})); + +interface SoundState { + sound: Audio.Sound | null; // Define the sound type + setSound: (newSound: Audio.Sound | null) => void; // Function to set sound +} + +const useSoundStore = create((set) => ({ + sound: null, // initial state + setSound: (newSound) => set({ sound: newSound }), // action to set sound +})); + const Main: React.FC = ({ route }) => { const { scannedData } = route.params; const [connectionStatus, setConnectionStatus] = useState("Connecting..."); const [ws, setWs] = useState(null); const [recording, setRecording] = useState(null); - const [audioQueue, setAudioQueue] = useState([]); - const [sound, setSound] = useState(); + const addToQueue = useAudioQueueStore((state) => state.addToQueue); + const audioQueue = useAudioQueueStore((state) => state.audioQueue); + const setSound = useSoundStore((state) => state.setSound); + const sound = useSoundStore((state) => state.sound); const audioDir = FileSystem.documentDirectory + '01/audio/'; const [permissionResponse, requestPermission] = Audio.usePermissions(); polyfillEncoding(); - const reader = new FileReader(); const constructTempFilePath = async (buffer: string) => { await dirExists(); const tempFilePath = `${audioDir}${Date.now()}.wav`; - await FileSystem.writeAsStringAsync( tempFilePath, buffer, @@ -37,7 +58,6 @@ const Main: React.FC = ({ route }) => { } ); - return tempFilePath; }; @@ -54,15 +74,10 @@ const Main: React.FC = ({ route }) => { } const playNextAudio = async () => { - console.log("in playNextAudio audioQueue is", audioQueue.length); + console.log(`in playNextAudio audioQueue is ${audioQueue.length} and sound is ${sound}`); - if (sound != null){ - console.log('Unloading Sound'); - await sound.unloadAsync(); - setSound(null); - } - if (audioQueue.length > 0) { + if (audioQueue.length > 0 && sound == null) { const uri = audioQueue.shift() as string; console.log("load audio from", uri); @@ -80,21 +95,32 @@ const Main: React.FC = ({ route }) => { playNextAudio(); } + } else { + console.log("audioQueue is empty or sound is not null"); + return; } }; + const _onPlayBackStatusUpdate = async (status: AVPlaybackStatus) => { + if (isAVPlaybackStatusSuccess(status) && status.didJustFinish === true){ + console.log("on playback status update sound is ", sound); + if (sound != null){ + console.log('Unloading Sound'); + await sound.unloadAsync(); + } + setSound(null); + console.log("audio has finished playing, playing next audio"); + console.log(audioQueue); + playNextAudio(); + } + } + const isAVPlaybackStatusSuccess = ( status: AVPlaybackStatus ): status is AVPlaybackStatusSuccess => { return (status as AVPlaybackStatusSuccess).isLoaded !== undefined; }; - const _onPlayBackStatusUpdate = (status: AVPlaybackStatus) => { - if (isAVPlaybackStatusSuccess(status) && status.didJustFinish){ - playNextAudio(); - } - } - useEffect(() => { console.log("audioQueue has been updated:", audioQueue.length); if (audioQueue.length == 1) { @@ -102,6 +128,10 @@ const Main: React.FC = ({ route }) => { } }, [audioQueue]); + useEffect(() => { + console.log("sound has been updated:", sound); + }, [sound]); + useEffect(() => { let websocket: WebSocket; try { @@ -121,13 +151,8 @@ const Main: React.FC = ({ route }) => { const buffer = await message.content as string; const filePath = await constructTempFilePath(buffer); - setAudioQueue((prevQueue) => [...prevQueue, filePath]); + addToQueue(filePath); console.log("audio file written to", filePath); - - if (message.format === "bytes.raw" && message.end && audioQueue.length >= 1) { - console.log("calling playNextAudio"); - playNextAudio(); - } }; websocket.onerror = (error) => { @@ -209,6 +234,7 @@ const Main: React.FC = ({ route }) => { console.log("fetched audio file", response); const blob = await response.blob(); + const reader = new FileReader(); reader.readAsArrayBuffer(blob); reader.onloadend = () => { const audioBytes = reader.result;