From f6c13a13bd0fa9d695d3d3f2d3ffccadd52b8399 Mon Sep 17 00:00:00 2001
From: Ben Xu <benx.xu@mail.utoronto.ca>
Date: Mon, 9 Dec 2024 13:35:55 -0800
Subject: [PATCH] send multimodal message on startup of multimodal agent

---
 software/source/server/livekit/multimodal.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/software/source/server/livekit/multimodal.py b/software/source/server/livekit/multimodal.py
index a9fe1f3..d1c9d4a 100644
--- a/software/source/server/livekit/multimodal.py
+++ b/software/source/server/livekit/multimodal.py
@@ -104,6 +104,9 @@ async def entrypoint(ctx: JobContext):
         fnc_ctx=fnc_ctx,
     )
 
+    # Send a message to the client to switch to multimodal mode
+    await ctx.room.local_participant.publish_data(payload="{MULTIMODAL_MODE}", topic="agent_mode")
+
     # Initial message to start the interaction
     session.conversation.item.create(
       llm.ChatMessage(