upload file to gcloud storage

software/source/clients/ios/react-native/src/screens/Main.tsx

@@ -19,6 +19,37 @@ const Main: React.FC<MainProps> = ({ route }) => {
   const [audioQueue, setAudioQueue] = useState<string[]>([]);
   const [sound, setSound] = useState<Audio.Sound | null>();
   const audioDir = FileSystem.documentDirectory + '01/audio/';
+  const Buffer = require('buffer').Buffer;
+
+  const toBuffer = async (blob: Blob) => {
+
+    const uri = await toDataURI(blob);
+    const base64 = uri.replace(/^.*,/g, "");
+    return Buffer.from(base64, "base64");
+  };
+
+  const toDataURI = (blob: Blob) =>
+    new Promise((resolve) => {
+      const reader = new FileReader();
+      reader.readAsDataURL(blob);
+      reader.onloadend = () => {
+        const uri = reader.result?.toString();
+        resolve(uri);
+      };
+    });
+
+    const constructTempFilePath = async (buffer: Buffer) => {
+      const tempFilePath = `${audioDir}${Date.now()}.wav`;
+      await FileSystem.writeAsStringAsync(
+        tempFilePath,
+        buffer.toString(),
+        {
+          encoding: FileSystem.EncodingType.Base64,
+        }
+      );
+
+      return tempFilePath;
+    };
 
 
   async function dirExists() {
@@ -33,7 +64,6 @@ const Main: React.FC<MainProps> = ({ route }) => {
   }
 
   const playNextAudio = async () => {
-    await dirExists();
     console.log("in playNextAudio audioQueue is", audioQueue.length);
 
     if (audioQueue.length > 0) {
@@ -65,6 +95,12 @@ const Main: React.FC<MainProps> = ({ route }) => {
       : undefined;
   }, [sound]);
 
+  useEffect(() => {
+    console.log("audioQueue has been updated:", audioQueue.length);
+    if (audioQueue.length == 1) {
+      playNextAudio();
+    }
+  }, [audioQueue]);
 
   useEffect(() => {
     let websocket: WebSocket;
@@ -79,31 +115,38 @@ const Main: React.FC<MainProps> = ({ route }) => {
       };
 
       websocket.onmessage = async (e) => {
+        console.log("Received message from WebSocket", e.data);
+
+        const blob = await e.data;
+        const buffer = await toBuffer(blob);
+        const filePath = await constructTempFilePath(buffer);
+        setAudioQueue((prevQueue) => [...prevQueue, filePath]);
+        console.log("audio file written to", filePath);
+
+        if (e.data.format === "bytes.raw" && e.data.end && audioQueue.length > 1) {
+          console.log("calling playNextAudio");
+          playNextAudio();
+        }
+
+        /**
         const message = JSON.parse(e.data);
 
         if (message.content) {
           const parsedMessage = message.content.replace(/^b'|['"]|['"]$/g, "");
           console.log("parsedMessage", parsedMessage.slice(0, 30));
 
-          const filePath = `${audioDir}${Date.now()}.mp3`;
-          await FileSystem.writeAsStringAsync(
-            filePath,
-            parsedMessage,
-            {
-              encoding: FileSystem.EncodingType.Base64,
-            }
-          );
-
-          console.log("audio file written to", filePath);
-
+          const filePath = await constructFilePath(parsedMessage);
           setAudioQueue((prevQueue) => [...prevQueue, filePath]);
+          console.log("audio file written to", filePath);
         }
 
-        if (message.format === "bytes.raw" && message.end) {
+        if (message.format === "bytes.raw" && message.end && audioQueue.length > 1) {
           console.log("calling playNextAudio");
           playNextAudio();
         }
 
+         */
+
       };
 
       websocket.onerror = (error) => {

software/source/server/server.py

@@ -21,6 +21,7 @@ from ..utils.accumulator import Accumulator
 from .utils.logs import setup_logging
 from .utils.logs import logger
 import base64
+from google.cloud import storage
 
 from ..utils.print_markdown import print_markdown
 
@@ -202,6 +203,10 @@ async def send_messages(websocket: WebSocket):
                 await websocket.send_json(message)
             elif isinstance(message, bytes):
                 message = base64.b64encode(message)
+                print(f"Sending to the device: {type(message)} {str(message)[:100]}")
+                await websocket.send_bytes(message)
+
+                """
                 str_bytes = str(message)
                 json_bytes = {
                     "role": "assistant",
@@ -213,6 +218,7 @@ async def send_messages(websocket: WebSocket):
                     f"Sending to the device: {type(json_bytes)} {str(json_bytes)[:100]}"
                 )
                 await websocket.send_json(json_bytes)
+                """
             else:
                 raise TypeError("Message must be a dict or bytes")
         except:
@@ -254,6 +260,7 @@ async def listener():
                 # Format will be bytes.wav or bytes.opus
                 mime_type = "audio/" + message["format"].split(".")[1]
                 audio_file_path = bytes_to_wav(message["content"], mime_type)
+                print("Audio file path:", audio_file_path)
 
                 # For microphone debugging:
                 if False:
@@ -387,6 +394,19 @@ def stream_tts(sentence):
 
     with open(audio_file, "rb") as f:
         audio_bytes = f.read()
+
+    storage_client = storage.Client(project="react-native-421323")
+    bucket = storage_client.bucket("01-audio")
+    blob = bucket.blob(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
+    generation_match_precondition = 0
+
+    blob.upload_from_filename(
+        audio_file, if_generation_match=generation_match_precondition
+    )
+    print(
+        f"Audio file {audio_file} uploaded to {datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav"
+    )
+
     os.remove(audio_file)
 
     file_type = "bytes.raw"