open-interpreter-ai
/
01
mirror of https://github.com/OpenInterpreter/01
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

streaming audio / playback

Browse Source
pull/45/head
Theia Vogel 11 months ago
parent 15bb6c92ac
commit 22761d0344
2 changed files with 99 additions and 41 deletions
Show Stats Download Patch File Download Diff File

90
01OS/01OS/clients/esp32/playback.ino → 01OS/01OS/clients/esp32/playback/playback.ino
Unescape View File

@ -11,6 +11,9 @@
#include <WebSocketsClient.h>
#define COMPUTER_IP "192.168.68.87"
#define CONFIG_I2S_BCK_PIN 19
#define CONFIG_I2S_LRCK_PIN 33
#define CONFIG_I2S_DATA_PIN 22
@ -22,13 +25,36 @@
#define MODE_SPK 1
#define DATA_SIZE 1024
uint8_t microphonedata0[1024 * 30];
uint8_t speakerdata0[1024 * 30];
uint8_t microphonedata0[1024 * 10];
uint8_t speakerdata0[1024 * 1];
int speaker_offset = 0;
int data_offset = 0;
WebSocketsClient webSocket;
class ButtonChecker {
public:
void loop() {
lastTickState = thisTickState;
thisTickState = M5.Btn.isPressed() != 0;
}
bool justPressed() {
return thisTickState && !lastTickState;
}
bool justReleased() {
return !thisTickState && lastTickState;
}
private:
bool lastTickState = false;
bool thisTickState = false;
};
ButtonChecker button = ButtonChecker();
void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) {
const uint8_t* src = (const uint8_t*) mem;
@ -43,8 +69,6 @@ void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) {
Serial.printf("\n");
}
void InitI2SSpeakerOrMic(int mode) {
esp_err_t err = ESP_OK;
@ -101,7 +125,6 @@ void speaker_play(uint8_t *payload, uint32_t len){
}
void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) {
switch (type) {
case WStype_DISCONNECTED:
Serial.printf("[WSc] Disconnected!\n");
@ -110,17 +133,18 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) {
Serial.printf("[WSc] Connected to url: %s\n", payload);
// send message to server when Connected
webSocket.sendTXT("Connected");
break;
case WStype_TEXT:
Serial.printf("[WSc] get text: %s\n", payload);
if ((char)payload[0] == 's'){
Serial.println("start");
speaker_offset = 0;
InitI2SSpeakerOrMic(MODE_SPK);
}
if ((char)payload[0] == 'e'){
Serial.println("end");
speaker_play(payload, speaker_offset);
// speaker_play(speakerdata0, speaker_offset);
// speaker_offset = 0;
}
// send message to server
@ -128,8 +152,11 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) {
break;
case WStype_BIN:
Serial.printf("[WSc] get binary length: %u\n", length);
memcpy(&(speakerdata0[speaker_offset]),&payload,length); // this line is likely the issue, the payloads here don't match the data that speaker_play contains
memcpy(speakerdata0 + speaker_offset, payload, length);
speaker_offset += length;
size_t bytes_written;
i2s_write(SPEAKER_I2S_NUMBER, speakerdata0, speaker_offset, &bytes_written, portMAX_DELAY);
speaker_offset = 0;
// send data to server
@ -153,7 +180,7 @@ void websocket_setup() {
Serial.println("connecting to WiFi");
}
Serial.println("connected to WiFi");
webSocket.begin("192.168.68.71", 9001, "/");
webSocket.begin(COMPUTER_IP, 9001, "/");
webSocket.onEvent(webSocketEvent);
// webSocket.setAuthorization("user", "Password");
webSocket.setReconnectInterval(5000);
@ -168,26 +195,43 @@ void setup() {
delay(2000);
}
bool recording = false;
void loop() {
if (M5.Btn.isPressed()) {
void flush_microphone() {
Serial.printf("[microphone] flushing %d bytes of data\n", data_offset);
if (data_offset == 0) return;
webSocket.sendBIN(microphonedata0, data_offset);
data_offset = 0;
InitI2SSpeakerOrMic(MODE_MIC);
M5.dis.drawpix(0, CRGB(128, 128, 0));
size_t byte_read;
}
while (1) {
i2s_read(SPEAKER_I2S_NUMBER,
(char *)(microphonedata0 + data_offset), DATA_SIZE,
&byte_read, (100 / portTICK_RATE_MS));
data_offset += 1024;
M5.update();
if (M5.Btn.isReleased() || data_offset >= 71679) break;
// delay(60);
void loop() {
button.loop();
if (button.justPressed()) {
Serial.println("Recording...");
webSocket.sendTXT("s");
InitI2SSpeakerOrMic(MODE_MIC);
recording = true;
} else if (button.justReleased()) {
Serial.println("Stopped recording.");
webSocket.sendTXT("e");
flush_microphone();
recording = false;
}
webSocket.sendBIN(microphonedata0, data_offset);
if (recording) {
size_t bytes_read;
i2s_read(
SPEAKER_I2S_NUMBER,
(char *)(microphonedata0 + data_offset),
DATA_SIZE, &bytes_read, (100 / portTICK_RATE_MS)
);
data_offset += bytes_read;
if (data_offset > 1024*10) {
flush_microphone();
}
}
M5.update();
webSocket.loop();

38
01OS/01OS/clients/esp32/websocket_test.py
Unescape View File

@ -1,8 +1,12 @@
#!/usr/bin/env python
"""A basic echo server for testing the device."""
import asyncio
import simpleaudio as sa
import uuid
import websockets
from websockets.server import serve
import traceback
def divide_chunks(l, n):
@ -11,23 +15,33 @@ def divide_chunks(l, n):
yield l[i : i + n]
async def echo(websocket):
async for message in websocket:
try:
play_obj = sa.play_buffer(bytearray(message), 1, 2, 16000)
play_obj.wait_done()
x = list(divide_chunks(bytearray(message), 1000))
for i in x:
await websocket.send(i)
buffers: dict[uuid.UUID, bytearray] = {}
except Exception as e:
print('fail')
async def echo(websocket: websockets.WebSocketServerProtocol):
async for message in websocket:
try:
if message == "s":
print("starting stream for", websocket.id)
buffers[websocket.id] = bytearray()
elif message == "e":
print("end, echoing stream for", websocket.id)
await websocket.send("s")
for chunk in divide_chunks(buffers[websocket.id], 1000):
await websocket.send(chunk)
await websocket.send("e")
elif type(message) is bytes:
print("recvd", len(message), "bytes from", websocket.id)
buffers[websocket.id].extend(message)
else:
print("ERR: recvd unknown message", message[:10], "from", websocket.id)
except Exception as _e:
traceback.print_exc()
async def main():
async with serve(echo, "0.0.0.0", 9001):
await asyncio.Future() # run forever
asyncio.run(main())
Write Preview
Loading…
Cancel
Save