From 72706c37a41941b3cb925dd8deb6c1149cac2e71 Mon Sep 17 00:00:00 2001 From: Shiven Mian Date: Sat, 17 Feb 2024 16:54:54 -0800 Subject: [PATCH 01/14] feat: added WiFi captive portal --- .../01OS/clients/esp32/wifi_captiveportal.ino | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 01OS/01OS/clients/esp32/wifi_captiveportal.ino diff --git a/01OS/01OS/clients/esp32/wifi_captiveportal.ino b/01OS/01OS/clients/esp32/wifi_captiveportal.ino new file mode 100644 index 0000000..2c772c2 --- /dev/null +++ b/01OS/01OS/clients/esp32/wifi_captiveportal.ino @@ -0,0 +1,190 @@ +#include //not needed in the arduino ide + +// Captive Portal +#include //https://github.com/me-no-dev/AsyncTCP using the latest dev version from @me-no-dev +#include +#include //https://github.com/me-no-dev/ESPAsyncWebServer using the latest dev version from @me-no-dev +#include //Used for mpdu_rx_disable android workaround + +// Pre reading on the fundamentals of captive portals https://textslashplain.com/2022/06/24/captive-portals/ + +const char *ssid = "captive"; // FYI The SSID can't have a space in it. +// const char * password = "12345678"; //Atleast 8 chars +const char *password = NULL; // no password + +#define MAX_CLIENTS 4 // ESP32 supports up to 10 but I have not tested it yet +#define WIFI_CHANNEL 6 // 2.4ghz channel 6 https://en.wikipedia.org/wiki/List_of_WLAN_channels#2.4_GHz_(802.11b/g/n/ax) + +const IPAddress localIP(4, 3, 2, 1); // the IP address the web server, Samsung requires the IP to be in public space +const IPAddress gatewayIP(4, 3, 2, 1); // IP address of the network should be the same as the local IP for captive portals +const IPAddress subnetMask(255, 255, 255, 0); // no need to change: https://avinetworks.com/glossary/subnet-mask/ + +const String localIPURL = "http://4.3.2.1"; // a string version of the local IP with http, used for redirecting clients to your webpage + +const char index_html[] PROGMEM = R"=====( + + + + ESP32 WiFi Setup + + + + +

WiFi Setup

+
+
+
+
+

+ +
+ + +)====="; + +DNSServer dnsServer; +AsyncWebServer server(80); + +void setUpDNSServer(DNSServer &dnsServer, const IPAddress &localIP) { +// Define the DNS interval in milliseconds between processing DNS requests +#define DNS_INTERVAL 30 + + // Set the TTL for DNS response and start the DNS server + dnsServer.setTTL(3600); + dnsServer.start(53, "*", localIP); +} + +void startSoftAccessPoint(const char *ssid, const char *password, const IPAddress &localIP, const IPAddress &gatewayIP) { +// Define the maximum number of clients that can connect to the server +#define MAX_CLIENTS 4 +// Define the WiFi channel to be used (channel 6 in this case) +#define WIFI_CHANNEL 6 + + // Set the WiFi mode to access point and station + WiFi.mode(WIFI_MODE_AP); + + // Define the subnet mask for the WiFi network + const IPAddress subnetMask(255, 255, 255, 0); + + // Configure the soft access point with a specific IP and subnet mask + WiFi.softAPConfig(localIP, gatewayIP, subnetMask); + + // Start the soft access point with the given ssid, password, channel, max number of clients + WiFi.softAP(ssid, password, WIFI_CHANNEL, 0, MAX_CLIENTS); + + // Disable AMPDU RX on the ESP32 WiFi to fix a bug on Android + esp_wifi_stop(); + esp_wifi_deinit(); + wifi_init_config_t my_config = WIFI_INIT_CONFIG_DEFAULT(); + my_config.ampdu_rx_enable = false; + esp_wifi_init(&my_config); + esp_wifi_start(); + vTaskDelay(100 / portTICK_PERIOD_MS); // Add a small delay +} + +void setUpWebserver(AsyncWebServer &server, const IPAddress &localIP) { + //======================== Webserver ======================== + // WARNING IOS (and maybe macos) WILL NOT POP UP IF IT CONTAINS THE WORD "Success" https://www.esp8266.com/viewtopic.php?f=34&t=4398 + // SAFARI (IOS) IS STUPID, G-ZIPPED FILES CAN'T END IN .GZ https://github.com/homieiot/homie-esp8266/issues/476 this is fixed by the webserver serve static function. + // SAFARI (IOS) there is a 128KB limit to the size of the HTML. The HTML can reference external resources/images that bring the total over 128KB + // SAFARI (IOS) popup browser has some severe limitations (javascript disabled, cookies disabled) + + // Required + server.on("/connecttest.txt", [](AsyncWebServerRequest *request) { request->redirect("http://logout.net"); }); // windows 11 captive portal workaround + server.on("/wpad.dat", [](AsyncWebServerRequest *request) { request->send(404); }); // Honestly don't understand what this is but a 404 stops win 10 keep calling this repeatedly and panicking the esp32 :) + + // Background responses: Probably not all are Required, but some are. Others might speed things up? + // A Tier (commonly used by modern systems) + server.on("/generate_204", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // android captive portal redirect + server.on("/redirect", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // microsoft redirect + server.on("/hotspot-detect.html", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // apple call home + server.on("/canonical.html", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // firefox captive portal call home + server.on("/success.txt", [](AsyncWebServerRequest *request) { request->send(200); }); // firefox captive portal call home + server.on("/ncsi.txt", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // windows call home + + // B Tier (uncommon) + // server.on("/chrome-variations/seed",[](AsyncWebServerRequest *request){request->send(200);}); //chrome captive portal call home + // server.on("/service/update2/json",[](AsyncWebServerRequest *request){request->send(200);}); //firefox? + // server.on("/chat",[](AsyncWebServerRequest *request){request->send(404);}); //No stop asking Whatsapp, there is no internet connection + // server.on("/startpage",[](AsyncWebServerRequest *request){request->redirect(localIPURL);}); + + // return 404 to webpage icon + server.on("/favicon.ico", [](AsyncWebServerRequest *request) { request->send(404); }); // webpage icon + + // Serve Basic HTML Page + server.on("/", HTTP_ANY, [](AsyncWebServerRequest *request) { + AsyncWebServerResponse *response = request->beginResponse(200, "text/html", index_html); + response->addHeader("Cache-Control", "public,max-age=31536000"); // save this file to cache for 1 year (unless you refresh) + request->send(response); + Serial.println("Served Basic HTML Page"); + }); + + server.on("/setup", HTTP_POST, [](AsyncWebServerRequest *request) { + String ssid; + String password; + if (request->hasParam("ssid", true) && request->hasParam("password", true)) { + ssid = request->getParam("ssid", true)->value(); + password = request->getParam("password", true)->value(); + // Now connect to the WiFi network using the provided credentials + WiFi.begin(ssid.c_str(), password.c_str()); + // You may want to add some logic to check if the connection was successful + // and respond accordingly + request->send(200, "text/plain", "Connecting to " + ssid); + } else { + request->send(400, "text/plain", "Invalid request"); + } + }); + + // the catch all + server.onNotFound([](AsyncWebServerRequest *request) { + request->redirect(localIPURL); + Serial.print("onnotfound "); + Serial.print(request->host()); // This gives some insight into whatever was being requested on the serial monitor + Serial.print(" "); + Serial.print(request->url()); + Serial.print(" sent redirect to " + localIPURL + "\n"); + }); +} + +void setup() { + // Set the transmit buffer size for the Serial object and start it with a baud rate of 115200. + Serial.setTxBufferSize(1024); + Serial.begin(115200); + + // Wait for the Serial object to become available. + while (!Serial) + ; + + // Print a welcome message to the Serial port. + Serial.println("\n\nCaptive Test, V0.5.0 compiled " __DATE__ " " __TIME__ " by CD_FER"); //__DATE__ is provided by the platformio ide + Serial.printf("%s-%d\n\r", ESP.getChipModel(), ESP.getChipRevision()); + + startSoftAccessPoint(ssid, password, localIP, gatewayIP); + + setUpDNSServer(dnsServer, localIP); + + setUpWebserver(server, localIP); + server.begin(); + + Serial.print("\n"); + Serial.print("Startup Time:"); // should be somewhere between 270-350 for Generic ESP32 (D0WDQ6 chip, can have a higher startup time on first boot) + Serial.println(millis()); + Serial.print("\n"); +} + +void loop() { + dnsServer.processNextRequest(); // I call this atleast every 10ms in my other projects (can be higher but I haven't tested it for stability) + delay(DNS_INTERVAL); // seems to help with stability, if you are doing other things in the loop this may not be needed + + // Check WiFi connection status + if (WiFi.status() == WL_CONNECTED) { + // If connected, you might want to do something, like printing the IP address + Serial.println("Connected to WiFi!"); + Serial.println("IP Address: " + WiFi.localIP().toString()); + Serial.println("SSID " + WiFi.SSID()); + } +} \ No newline at end of file From b4dd20950bba235a8b3c167c9ac327b1ab6eb277 Mon Sep 17 00:00:00 2001 From: Lincoln Roth Date: Sat, 17 Feb 2024 18:17:49 -0800 Subject: [PATCH 02/14] websockets from esp32 to server work, very chunky in other direction --- 01OS/01OS/clients/esp32/playback.ino | 110 ++++++++++++++++++++-- 01OS/01OS/clients/esp32/websocket_test.py | 33 +++++++ 2 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 01OS/01OS/clients/esp32/websocket_test.py diff --git a/01OS/01OS/clients/esp32/playback.ino b/01OS/01OS/clients/esp32/playback.ino index 61c800c..3dc91c1 100644 --- a/01OS/01OS/clients/esp32/playback.ino +++ b/01OS/01OS/clients/esp32/playback.ino @@ -3,6 +3,14 @@ #include #include +#include + +#include +#include +#include + +#include + #define CONFIG_I2S_BCK_PIN 19 #define CONFIG_I2S_LRCK_PIN 33 #define CONFIG_I2S_DATA_PIN 22 @@ -17,6 +25,24 @@ uint8_t microphonedata0[1024 * 70]; int data_offset = 0; +WebSocketsClient webSocket; + + +void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) { + const uint8_t* src = (const uint8_t*) mem; + Serial.printf("\n[HEXDUMP] Address: 0x%08X len: 0x%X (%d)", (ptrdiff_t)src, len, len); + for (uint32_t i = 0; i < len; i++) { + if (i % cols == 0) { + Serial.printf("\n[0x%08X] 0x%08X: ", (ptrdiff_t)src, i); + } + Serial.printf("%02X ", *src); + src++; + } + Serial.printf("\n"); +} + + + void InitI2SSpeakerOrMic(int mode) { esp_err_t err = ESP_OK; @@ -25,11 +51,11 @@ void InitI2SSpeakerOrMic(int mode) { .mode = (i2s_mode_t)(I2S_MODE_MASTER), .sample_rate = 16000, .bits_per_sample = - I2S_BITS_PER_SAMPLE_16BIT, // is fixed at 12bit, stereo, MSB + I2S_BITS_PER_SAMPLE_16BIT, // is fixed at 12bit, stereo, MSB .channel_format = I2S_CHANNEL_FMT_ALL_RIGHT, #if ESP_IDF_VERSION > ESP_IDF_VERSION_VAL(4, 1, 0) .communication_format = - I2S_COMM_FORMAT_STAND_I2S, // Set the format of the communication. + I2S_COMM_FORMAT_STAND_I2S, // Set the format of the communication. #else // 设置通讯格式 .communication_format = I2S_COMM_FORMAT_I2S, #endif @@ -64,12 +90,71 @@ void InitI2SSpeakerOrMic(int mode) { I2S_CHANNEL_MONO); } +void speaker_play(uint8_t *payload, uint32_t len){ + size_t bytes_written; + InitI2SSpeakerOrMic(MODE_SPK); + i2s_write(SPEAKER_I2S_NUMBER, payload, len, + &bytes_written, portMAX_DELAY); +} + +void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { + + switch (type) { + case WStype_DISCONNECTED: + Serial.printf("[WSc] Disconnected!\n"); + break; + case WStype_CONNECTED: + Serial.printf("[WSc] Connected to url: %s\n", payload); + + // send message to server when Connected + webSocket.sendTXT("Connected"); + break; + case WStype_TEXT: + Serial.printf("[WSc] get text: %s\n", payload); + + // send message to server + // webSocket.sendTXT("message here"); + break; + case WStype_BIN: + Serial.printf("[WSc] get binary length: %u\n", length); + // hexdump(payload, length); + speaker_play(payload, length); + + // send data to server + // webSocket.sendBIN(payload, length); + break; + case WStype_ERROR: + case WStype_FRAGMENT_TEXT_START: + case WStype_FRAGMENT_BIN_START: + case WStype_FRAGMENT: + case WStype_FRAGMENT_FIN: + break; + } + +} + +void websocket_setup() { + Serial.begin(115200); + WiFi.begin("Soundview_Guest", ""); + while (WiFi.status() != WL_CONNECTED){ + delay(500); + Serial.println("connecting to WiFi"); + } + Serial.println("connected to WiFi"); + webSocket.begin("192.168.68.71", 9001, "/"); + webSocket.onEvent(webSocketEvent); + // webSocket.setAuthorization("user", "Password"); + webSocket.setReconnectInterval(5000); + +} void setup() { M5.begin(true, false, true); M5.dis.drawpix(0, CRGB(128, 128, 0)); + websocket_setup(); delay(2000); } + void loop() { if (M5.Btn.isPressed()) { data_offset = 0; @@ -86,10 +171,21 @@ void loop() { if (M5.Btn.isReleased() || data_offset >= 71679) break; // delay(60); } - size_t bytes_written; - InitI2SSpeakerOrMic(MODE_SPK); - i2s_write(SPEAKER_I2S_NUMBER, microphonedata0, data_offset, - &bytes_written, portMAX_DELAY); + + Serial.println(data_offset); + + int chunk_size = 1004; + chunk_size = 4096; + + webSocket.sendBIN(microphonedata0, data_offset); + + // size_t bytes_written; + // InitI2SSpeakerOrMic(MODE_SPK); + // i2s_write(SPEAKER_I2S_NUMBER, microphonedata0, data_offset, + // &bytes_written, portMAX_DELAY); + } M5.update(); -} \ No newline at end of file + webSocket.loop(); + +} diff --git a/01OS/01OS/clients/esp32/websocket_test.py b/01OS/01OS/clients/esp32/websocket_test.py new file mode 100644 index 0000000..71c6813 --- /dev/null +++ b/01OS/01OS/clients/esp32/websocket_test.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import asyncio +import simpleaudio as sa +from websockets.server import serve + + +def divide_chunks(l, n): + # looping till length l + for i in range(0, len(l), n): + yield l[i:i + n] + + +async def echo(websocket): + async for message in websocket: + try: + play_obj = sa.play_buffer(bytearray(message), 1, 2, 16000) + play_obj.wait_done() + + x = list(divide_chunks(bytearray(message), 1000)) + for i in x: + await websocket.send(i) + + except Exception as e: + print('fail') + + + +async def main(): + async with serve(echo, "0.0.0.0", 9001): + await asyncio.Future() # run forever + +asyncio.run(main()) \ No newline at end of file From 7eb1cc0a1388af29af5f26e205c7bc3ca21c226e Mon Sep 17 00:00:00 2001 From: killian <63927363+KillianLucas@users.noreply.github.com> Date: Sat, 17 Feb 2024 18:54:37 -0800 Subject: [PATCH 03/14] Lenient teach mode, incremented version --- 01OS/01OS/clients/base_device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/01OS/01OS/clients/base_device.py b/01OS/01OS/clients/base_device.py index 7e6945b..a80fb8d 100644 --- a/01OS/01OS/clients/base_device.py +++ b/01OS/01OS/clients/base_device.py @@ -329,6 +329,6 @@ class Device: listener.start() def start(self): - if os.getenv('TEACH_MODE') == "False": + if os.getenv('TEACH_MODE') != "True": asyncio.run(self.start_async()) p.terminate() \ No newline at end of file From e9890bea801b057f8619a22c16bc332b29671538 Mon Sep 17 00:00:00 2001 From: killian <63927363+KillianLucas@users.noreply.github.com> Date: Sat, 17 Feb 2024 18:55:14 -0800 Subject: [PATCH 04/14] Lenient teach mode, incremented version --- 01OS/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/01OS/pyproject.toml b/01OS/pyproject.toml index 5f4d26c..83d0295 100644 --- a/01OS/pyproject.toml +++ b/01OS/pyproject.toml @@ -4,7 +4,7 @@ packages = [ {include = "01OS"}, ] include = [".env.example", "start.py", "start.sh"] -version = "0.0.3" +version = "0.0.4" description = "The open-source language model computer" authors = ["Killian "] license = "AGPL" From 25cf7c8284e7a14cc7a162ff93547563dd49d717 Mon Sep 17 00:00:00 2001 From: Zach Wener Date: Sat, 17 Feb 2024 18:58:22 -0800 Subject: [PATCH 05/14] feat: added wifi scanning --- .../01OS/clients/esp32/wifi_captiveportal.ino | 190 ------------- .../wifi_captiveportal/wifi_captiveportal.ino | 260 ++++++++++++++++++ 2 files changed, 260 insertions(+), 190 deletions(-) delete mode 100644 01OS/01OS/clients/esp32/wifi_captiveportal.ino create mode 100644 01OS/01OS/clients/esp32/wifi_captiveportal/wifi_captiveportal.ino diff --git a/01OS/01OS/clients/esp32/wifi_captiveportal.ino b/01OS/01OS/clients/esp32/wifi_captiveportal.ino deleted file mode 100644 index 2c772c2..0000000 --- a/01OS/01OS/clients/esp32/wifi_captiveportal.ino +++ /dev/null @@ -1,190 +0,0 @@ -#include //not needed in the arduino ide - -// Captive Portal -#include //https://github.com/me-no-dev/AsyncTCP using the latest dev version from @me-no-dev -#include -#include //https://github.com/me-no-dev/ESPAsyncWebServer using the latest dev version from @me-no-dev -#include //Used for mpdu_rx_disable android workaround - -// Pre reading on the fundamentals of captive portals https://textslashplain.com/2022/06/24/captive-portals/ - -const char *ssid = "captive"; // FYI The SSID can't have a space in it. -// const char * password = "12345678"; //Atleast 8 chars -const char *password = NULL; // no password - -#define MAX_CLIENTS 4 // ESP32 supports up to 10 but I have not tested it yet -#define WIFI_CHANNEL 6 // 2.4ghz channel 6 https://en.wikipedia.org/wiki/List_of_WLAN_channels#2.4_GHz_(802.11b/g/n/ax) - -const IPAddress localIP(4, 3, 2, 1); // the IP address the web server, Samsung requires the IP to be in public space -const IPAddress gatewayIP(4, 3, 2, 1); // IP address of the network should be the same as the local IP for captive portals -const IPAddress subnetMask(255, 255, 255, 0); // no need to change: https://avinetworks.com/glossary/subnet-mask/ - -const String localIPURL = "http://4.3.2.1"; // a string version of the local IP with http, used for redirecting clients to your webpage - -const char index_html[] PROGMEM = R"=====( - - - - ESP32 WiFi Setup - - - - -

WiFi Setup

-
-
-
-
-

- -
- - -)====="; - -DNSServer dnsServer; -AsyncWebServer server(80); - -void setUpDNSServer(DNSServer &dnsServer, const IPAddress &localIP) { -// Define the DNS interval in milliseconds between processing DNS requests -#define DNS_INTERVAL 30 - - // Set the TTL for DNS response and start the DNS server - dnsServer.setTTL(3600); - dnsServer.start(53, "*", localIP); -} - -void startSoftAccessPoint(const char *ssid, const char *password, const IPAddress &localIP, const IPAddress &gatewayIP) { -// Define the maximum number of clients that can connect to the server -#define MAX_CLIENTS 4 -// Define the WiFi channel to be used (channel 6 in this case) -#define WIFI_CHANNEL 6 - - // Set the WiFi mode to access point and station - WiFi.mode(WIFI_MODE_AP); - - // Define the subnet mask for the WiFi network - const IPAddress subnetMask(255, 255, 255, 0); - - // Configure the soft access point with a specific IP and subnet mask - WiFi.softAPConfig(localIP, gatewayIP, subnetMask); - - // Start the soft access point with the given ssid, password, channel, max number of clients - WiFi.softAP(ssid, password, WIFI_CHANNEL, 0, MAX_CLIENTS); - - // Disable AMPDU RX on the ESP32 WiFi to fix a bug on Android - esp_wifi_stop(); - esp_wifi_deinit(); - wifi_init_config_t my_config = WIFI_INIT_CONFIG_DEFAULT(); - my_config.ampdu_rx_enable = false; - esp_wifi_init(&my_config); - esp_wifi_start(); - vTaskDelay(100 / portTICK_PERIOD_MS); // Add a small delay -} - -void setUpWebserver(AsyncWebServer &server, const IPAddress &localIP) { - //======================== Webserver ======================== - // WARNING IOS (and maybe macos) WILL NOT POP UP IF IT CONTAINS THE WORD "Success" https://www.esp8266.com/viewtopic.php?f=34&t=4398 - // SAFARI (IOS) IS STUPID, G-ZIPPED FILES CAN'T END IN .GZ https://github.com/homieiot/homie-esp8266/issues/476 this is fixed by the webserver serve static function. - // SAFARI (IOS) there is a 128KB limit to the size of the HTML. The HTML can reference external resources/images that bring the total over 128KB - // SAFARI (IOS) popup browser has some severe limitations (javascript disabled, cookies disabled) - - // Required - server.on("/connecttest.txt", [](AsyncWebServerRequest *request) { request->redirect("http://logout.net"); }); // windows 11 captive portal workaround - server.on("/wpad.dat", [](AsyncWebServerRequest *request) { request->send(404); }); // Honestly don't understand what this is but a 404 stops win 10 keep calling this repeatedly and panicking the esp32 :) - - // Background responses: Probably not all are Required, but some are. Others might speed things up? - // A Tier (commonly used by modern systems) - server.on("/generate_204", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // android captive portal redirect - server.on("/redirect", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // microsoft redirect - server.on("/hotspot-detect.html", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // apple call home - server.on("/canonical.html", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // firefox captive portal call home - server.on("/success.txt", [](AsyncWebServerRequest *request) { request->send(200); }); // firefox captive portal call home - server.on("/ncsi.txt", [](AsyncWebServerRequest *request) { request->redirect(localIPURL); }); // windows call home - - // B Tier (uncommon) - // server.on("/chrome-variations/seed",[](AsyncWebServerRequest *request){request->send(200);}); //chrome captive portal call home - // server.on("/service/update2/json",[](AsyncWebServerRequest *request){request->send(200);}); //firefox? - // server.on("/chat",[](AsyncWebServerRequest *request){request->send(404);}); //No stop asking Whatsapp, there is no internet connection - // server.on("/startpage",[](AsyncWebServerRequest *request){request->redirect(localIPURL);}); - - // return 404 to webpage icon - server.on("/favicon.ico", [](AsyncWebServerRequest *request) { request->send(404); }); // webpage icon - - // Serve Basic HTML Page - server.on("/", HTTP_ANY, [](AsyncWebServerRequest *request) { - AsyncWebServerResponse *response = request->beginResponse(200, "text/html", index_html); - response->addHeader("Cache-Control", "public,max-age=31536000"); // save this file to cache for 1 year (unless you refresh) - request->send(response); - Serial.println("Served Basic HTML Page"); - }); - - server.on("/setup", HTTP_POST, [](AsyncWebServerRequest *request) { - String ssid; - String password; - if (request->hasParam("ssid", true) && request->hasParam("password", true)) { - ssid = request->getParam("ssid", true)->value(); - password = request->getParam("password", true)->value(); - // Now connect to the WiFi network using the provided credentials - WiFi.begin(ssid.c_str(), password.c_str()); - // You may want to add some logic to check if the connection was successful - // and respond accordingly - request->send(200, "text/plain", "Connecting to " + ssid); - } else { - request->send(400, "text/plain", "Invalid request"); - } - }); - - // the catch all - server.onNotFound([](AsyncWebServerRequest *request) { - request->redirect(localIPURL); - Serial.print("onnotfound "); - Serial.print(request->host()); // This gives some insight into whatever was being requested on the serial monitor - Serial.print(" "); - Serial.print(request->url()); - Serial.print(" sent redirect to " + localIPURL + "\n"); - }); -} - -void setup() { - // Set the transmit buffer size for the Serial object and start it with a baud rate of 115200. - Serial.setTxBufferSize(1024); - Serial.begin(115200); - - // Wait for the Serial object to become available. - while (!Serial) - ; - - // Print a welcome message to the Serial port. - Serial.println("\n\nCaptive Test, V0.5.0 compiled " __DATE__ " " __TIME__ " by CD_FER"); //__DATE__ is provided by the platformio ide - Serial.printf("%s-%d\n\r", ESP.getChipModel(), ESP.getChipRevision()); - - startSoftAccessPoint(ssid, password, localIP, gatewayIP); - - setUpDNSServer(dnsServer, localIP); - - setUpWebserver(server, localIP); - server.begin(); - - Serial.print("\n"); - Serial.print("Startup Time:"); // should be somewhere between 270-350 for Generic ESP32 (D0WDQ6 chip, can have a higher startup time on first boot) - Serial.println(millis()); - Serial.print("\n"); -} - -void loop() { - dnsServer.processNextRequest(); // I call this atleast every 10ms in my other projects (can be higher but I haven't tested it for stability) - delay(DNS_INTERVAL); // seems to help with stability, if you are doing other things in the loop this may not be needed - - // Check WiFi connection status - if (WiFi.status() == WL_CONNECTED) { - // If connected, you might want to do something, like printing the IP address - Serial.println("Connected to WiFi!"); - Serial.println("IP Address: " + WiFi.localIP().toString()); - Serial.println("SSID " + WiFi.SSID()); - } -} \ No newline at end of file diff --git a/01OS/01OS/clients/esp32/wifi_captiveportal/wifi_captiveportal.ino b/01OS/01OS/clients/esp32/wifi_captiveportal/wifi_captiveportal.ino new file mode 100644 index 0000000..92bceaf --- /dev/null +++ b/01OS/01OS/clients/esp32/wifi_captiveportal/wifi_captiveportal.ino @@ -0,0 +1,260 @@ +#include //not needed in the arduino ide + +// Captive Portal +#include //https://github.com/me-no-dev/AsyncTCP using the latest dev version from @me-no-dev +#include +#include //https://github.com/me-no-dev/ESPAsyncWebServer using the latest dev version from @me-no-dev +#include //Used for mpdu_rx_disable android workaround + +// Pre reading on the fundamentals of captive portals https://textslashplain.com/2022/06/24/captive-portals/ + +const char *ssid = "captive"; // FYI The SSID can't have a space in it. +// const char * password = "12345678"; //Atleast 8 chars +const char *password = NULL; // no password + +#define MAX_CLIENTS 4 // ESP32 supports up to 10 but I have not tested it yet +#define WIFI_CHANNEL 6 // 2.4ghz channel 6 https://en.wikipedia.org/wiki/List_of_WLAN_channels#2.4_GHz_(802.11b/g/n/ax) + +const IPAddress localIP(4, 3, 2, 1); // the IP address the web server, Samsung requires the IP to be in public space +const IPAddress gatewayIP(4, 3, 2, 1); // IP address of the network should be the same as the local IP for captive portals +const IPAddress subnetMask(255, 255, 255, 0); // no need to change: https://avinetworks.com/glossary/subnet-mask/ + +const String localIPURL = "http://4.3.2.1"; // a string version of the local IP with http, used for redirecting clients to your webpage + +String generateHTMLWithSSIDs() +{ + String html = "

Select Wi-Fi Network



"; + + return html; +} + +const char index_html[] PROGMEM = R"=====( + + + + ESP32 WiFi Setup + + + + +

WiFi Setup

+
+
+
+
+

+ +
+ + +)====="; + +DNSServer dnsServer; +AsyncWebServer server(80); + +void setUpDNSServer(DNSServer &dnsServer, const IPAddress &localIP) +{ +// Define the DNS interval in milliseconds between processing DNS requests +#define DNS_INTERVAL 30 + + // Set the TTL for DNS response and start the DNS server + dnsServer.setTTL(3600); + dnsServer.start(53, "*", localIP); +} + +void startSoftAccessPoint(const char *ssid, const char *password, const IPAddress &localIP, const IPAddress &gatewayIP) +{ +// Define the maximum number of clients that can connect to the server +#define MAX_CLIENTS 4 +// Define the WiFi channel to be used (channel 6 in this case) +#define WIFI_CHANNEL 6 + + // Set the WiFi mode to access point and station + // WiFi.mode(WIFI_MODE_AP); + + // Define the subnet mask for the WiFi network + const IPAddress subnetMask(255, 255, 255, 0); + + // Configure the soft access point with a specific IP and subnet mask + WiFi.softAPConfig(localIP, gatewayIP, subnetMask); + + // Start the soft access point with the given ssid, password, channel, max number of clients + WiFi.softAP(ssid, password, WIFI_CHANNEL, 0, MAX_CLIENTS); + + // Disable AMPDU RX on the ESP32 WiFi to fix a bug on Android + esp_wifi_stop(); + esp_wifi_deinit(); + wifi_init_config_t my_config = WIFI_INIT_CONFIG_DEFAULT(); + my_config.ampdu_rx_enable = false; + esp_wifi_init(&my_config); + esp_wifi_start(); + vTaskDelay(100 / portTICK_PERIOD_MS); // Add a small delay +} + +void connectToWifi(String ssid, String password) +{ + WiFi.begin(ssid.c_str(), password.c_str()); + + // Wait for connection to establish + int attempts = 0; + while (WiFi.status() != WL_CONNECTED && attempts < 20) + { + delay(1000); + Serial.print("."); + attempts++; + } + + if (WiFi.status() == WL_CONNECTED) + { + Serial.println("Connected to Wi-Fi"); + } + else + { + Serial.println("Failed to connect to Wi-Fi. Check credentials."); + } +} + +void setUpWebserver(AsyncWebServer &server, const IPAddress &localIP) +{ + //======================== Webserver ======================== + // WARNING IOS (and maybe macos) WILL NOT POP UP IF IT CONTAINS THE WORD "Success" https://www.esp8266.com/viewtopic.php?f=34&t=4398 + // SAFARI (IOS) IS STUPID, G-ZIPPED FILES CAN'T END IN .GZ https://github.com/homieiot/homie-esp8266/issues/476 this is fixed by the webserver serve static function. + // SAFARI (IOS) there is a 128KB limit to the size of the HTML. The HTML can reference external resources/images that bring the total over 128KB + // SAFARI (IOS) popup browser has some severe limitations (javascript disabled, cookies disabled) + + // Required + server.on("/connecttest.txt", [](AsyncWebServerRequest *request) + { request->redirect("http://logout.net"); }); // windows 11 captive portal workaround + server.on("/wpad.dat", [](AsyncWebServerRequest *request) + { request->send(404); }); // Honestly don't understand what this is but a 404 stops win 10 keep calling this repeatedly and panicking the esp32 :) + + // Background responses: Probably not all are Required, but some are. Others might speed things up? + // A Tier (commonly used by modern systems) + server.on("/generate_204", [](AsyncWebServerRequest *request) + { request->redirect(localIPURL); }); // android captive portal redirect + server.on("/redirect", [](AsyncWebServerRequest *request) + { request->redirect(localIPURL); }); // microsoft redirect + server.on("/hotspot-detect.html", [](AsyncWebServerRequest *request) + { request->redirect(localIPURL); }); // apple call home + server.on("/canonical.html", [](AsyncWebServerRequest *request) + { request->redirect(localIPURL); }); // firefox captive portal call home + server.on("/success.txt", [](AsyncWebServerRequest *request) + { request->send(200); }); // firefox captive portal call home + server.on("/ncsi.txt", [](AsyncWebServerRequest *request) + { request->redirect(localIPURL); }); // windows call home + + // B Tier (uncommon) + // server.on("/chrome-variations/seed",[](AsyncWebServerRequest *request){request->send(200);}); //chrome captive portal call home + // server.on("/service/update2/json",[](AsyncWebServerRequest *request){request->send(200);}); //firefox? + // server.on("/chat",[](AsyncWebServerRequest *request){request->send(404);}); //No stop asking Whatsapp, there is no internet connection + // server.on("/startpage",[](AsyncWebServerRequest *request){request->redirect(localIPURL);}); + + // return 404 to webpage icon + server.on("/favicon.ico", [](AsyncWebServerRequest *request) + { request->send(404); }); // webpage icon + + // Serve Basic HTML Page + server.on("/", HTTP_ANY, [](AsyncWebServerRequest *request) + { + String htmlContent = index_html; + Serial.printf("wifi scan complete: %d . WIFI_SCAN_RUNNING: %d", WiFi.scanComplete(), WIFI_SCAN_RUNNING); + if(WiFi.scanComplete() > 0) { + // Scan complete, process results + Serial.println("done scanning wifi"); + htmlContent = generateHTMLWithSSIDs(); + // WiFi.scanNetworks(true); // Start a new scan in async mode + } + AsyncWebServerResponse *response = request->beginResponse(200, "text/html", htmlContent); + response->addHeader("Cache-Control", "public,max-age=31536000"); // save this file to cache for 1 year (unless you refresh) + request->send(response); + Serial.println("Served Basic HTML Page"); }); + + // the catch all + server.onNotFound([](AsyncWebServerRequest *request) + { + request->redirect(localIPURL); + Serial.print("onnotfound "); + Serial.print(request->host()); // This gives some insight into whatever was being requested on the serial monitor + Serial.print(" "); + Serial.print(request->url()); + Serial.print(" sent redirect to " + localIPURL + "\n"); }); + + server.on("/submit", HTTP_POST, [](AsyncWebServerRequest *request) + { + String ssid; + String password; + + // Check if SSID parameter exists and assign it + if(request->hasParam("ssid", true)) { + ssid = request->getParam("ssid", true)->value(); + } + + // Check if Password parameter exists and assign it + if(request->hasParam("password", true)) { + password = request->getParam("password", true)->value(); + } + + // Attempt to connect to the Wi-Fi network with these credentials + connectToWifi(ssid, password); + + // Redirect user or send a response back + request->send(200, "text/plain", "Attempting to connect to " + ssid); }); +} + +void setup() +{ + // Set the transmit buffer size for the Serial object and start it with a baud rate of 115200. + Serial.setTxBufferSize(1024); + Serial.begin(115200); + + // Wait for the Serial object to become available. + while (!Serial) + ; + + WiFi.mode(WIFI_AP_STA); + + // Print a welcome message to the Serial port. + Serial.println("\n\nCaptive Test, V0.5.0 compiled " __DATE__ " " __TIME__ " by CD_FER"); //__DATE__ is provided by the platformio ide + Serial.printf("%s-%d\n\r", ESP.getChipModel(), ESP.getChipRevision()); + + startSoftAccessPoint(ssid, password, localIP, gatewayIP); + + setUpDNSServer(dnsServer, localIP); + + WiFi.scanNetworks(true); + + setUpWebserver(server, localIP); + server.begin(); + + Serial.print("\n"); + Serial.print("Startup Time:"); // should be somewhere between 270-350 for Generic ESP32 (D0WDQ6 chip, can have a higher startup time on first boot) + Serial.println(millis()); + Serial.print("\n"); +} + +void loop() +{ + dnsServer.processNextRequest(); // I call this atleast every 10ms in my other projects (can be higher but I haven't tested it for stability) + delay(DNS_INTERVAL); // seems to help with stability, if you are doing other things in the loop this may not be needed + + // Check WiFi connection status + if (WiFi.status() == WL_CONNECTED) + { + // If connected, you might want to do something, like printing the IP address + Serial.println("Connected to WiFi!"); + Serial.println("IP Address: " + WiFi.localIP().toString()); + Serial.println("SSID " + WiFi.SSID()); + } +} \ No newline at end of file From 1ca19d396a1be361f1211abaddf96569b0f924a2 Mon Sep 17 00:00:00 2001 From: Lincoln Roth Date: Sat, 17 Feb 2024 19:42:26 -0800 Subject: [PATCH 06/14] works on debugging websocket receive --- 01OS/01OS/clients/esp32/playback.ino | 31 +++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/01OS/01OS/clients/esp32/playback.ino b/01OS/01OS/clients/esp32/playback.ino index 3dc91c1..228639e 100644 --- a/01OS/01OS/clients/esp32/playback.ino +++ b/01OS/01OS/clients/esp32/playback.ino @@ -22,7 +22,9 @@ #define MODE_SPK 1 #define DATA_SIZE 1024 -uint8_t microphonedata0[1024 * 70]; +uint8_t microphonedata0[1024 * 30]; +uint8_t speakerdata0[1024 * 30]; +int speaker_offset = 0; int data_offset = 0; WebSocketsClient webSocket; @@ -91,6 +93,7 @@ void InitI2SSpeakerOrMic(int mode) { } void speaker_play(uint8_t *payload, uint32_t len){ + Serial.printf("received %lu bytes", len); size_t bytes_written; InitI2SSpeakerOrMic(MODE_SPK); i2s_write(SPEAKER_I2S_NUMBER, payload, len, @@ -111,14 +114,23 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { break; case WStype_TEXT: Serial.printf("[WSc] get text: %s\n", payload); + if ((char)payload[0] == 's'){ + Serial.println("start"); + speaker_offset = 0; + } + if ((char)payload[0] == 'e'){ + Serial.println("end"); + speaker_play(payload, speaker_offset); + } // send message to server // webSocket.sendTXT("message here"); break; case WStype_BIN: Serial.printf("[WSc] get binary length: %u\n", length); - // hexdump(payload, length); - speaker_play(payload, length); + memcpy(&(speakerdata0[speaker_offset]),&payload,length); // this line is likely the issue, the payloads here don't match the data that speaker_play contains + speaker_offset += length; + // send data to server // webSocket.sendBIN(payload, length); @@ -151,6 +163,8 @@ void setup() { M5.begin(true, false, true); M5.dis.drawpix(0, CRGB(128, 128, 0)); websocket_setup(); + InitI2SSpeakerOrMic(MODE_SPK); + delay(2000); } @@ -171,19 +185,8 @@ void loop() { if (M5.Btn.isReleased() || data_offset >= 71679) break; // delay(60); } - - Serial.println(data_offset); - - int chunk_size = 1004; - chunk_size = 4096; - webSocket.sendBIN(microphonedata0, data_offset); - // size_t bytes_written; - // InitI2SSpeakerOrMic(MODE_SPK); - // i2s_write(SPEAKER_I2S_NUMBER, microphonedata0, data_offset, - // &bytes_written, portMAX_DELAY); - } M5.update(); webSocket.loop(); From 22761d0344ded27e0de1d94b3ed071f8dc086572 Mon Sep 17 00:00:00 2001 From: Theia Vogel Date: Sat, 17 Feb 2024 22:45:22 -0800 Subject: [PATCH 07/14] streaming audio / playback --- .../clients/esp32/{ => playback}/playback.ino | 92 ++++++++++++++----- 01OS/01OS/clients/esp32/websocket_test.py | 48 ++++++---- 2 files changed, 99 insertions(+), 41 deletions(-) rename 01OS/01OS/clients/esp32/{ => playback}/playback.ino (72%) diff --git a/01OS/01OS/clients/esp32/playback.ino b/01OS/01OS/clients/esp32/playback/playback.ino similarity index 72% rename from 01OS/01OS/clients/esp32/playback.ino rename to 01OS/01OS/clients/esp32/playback/playback.ino index 228639e..7f2e2a3 100644 --- a/01OS/01OS/clients/esp32/playback.ino +++ b/01OS/01OS/clients/esp32/playback/playback.ino @@ -11,6 +11,9 @@ #include + +#define COMPUTER_IP "192.168.68.87" + #define CONFIG_I2S_BCK_PIN 19 #define CONFIG_I2S_LRCK_PIN 33 #define CONFIG_I2S_DATA_PIN 22 @@ -22,13 +25,36 @@ #define MODE_SPK 1 #define DATA_SIZE 1024 -uint8_t microphonedata0[1024 * 30]; -uint8_t speakerdata0[1024 * 30]; +uint8_t microphonedata0[1024 * 10]; +uint8_t speakerdata0[1024 * 1]; int speaker_offset = 0; int data_offset = 0; WebSocketsClient webSocket; +class ButtonChecker { + public: + void loop() { + lastTickState = thisTickState; + thisTickState = M5.Btn.isPressed() != 0; + } + + bool justPressed() { + return thisTickState && !lastTickState; + } + + bool justReleased() { + return !thisTickState && lastTickState; + } + + private: + bool lastTickState = false; + bool thisTickState = false; +}; + +ButtonChecker button = ButtonChecker(); + + void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) { const uint8_t* src = (const uint8_t*) mem; @@ -43,8 +69,6 @@ void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) { Serial.printf("\n"); } - - void InitI2SSpeakerOrMic(int mode) { esp_err_t err = ESP_OK; @@ -101,7 +125,6 @@ void speaker_play(uint8_t *payload, uint32_t len){ } void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { - switch (type) { case WStype_DISCONNECTED: Serial.printf("[WSc] Disconnected!\n"); @@ -110,17 +133,18 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { Serial.printf("[WSc] Connected to url: %s\n", payload); // send message to server when Connected - webSocket.sendTXT("Connected"); break; case WStype_TEXT: Serial.printf("[WSc] get text: %s\n", payload); if ((char)payload[0] == 's'){ Serial.println("start"); speaker_offset = 0; + InitI2SSpeakerOrMic(MODE_SPK); } if ((char)payload[0] == 'e'){ Serial.println("end"); - speaker_play(payload, speaker_offset); + // speaker_play(speakerdata0, speaker_offset); + // speaker_offset = 0; } // send message to server @@ -128,8 +152,11 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { break; case WStype_BIN: Serial.printf("[WSc] get binary length: %u\n", length); - memcpy(&(speakerdata0[speaker_offset]),&payload,length); // this line is likely the issue, the payloads here don't match the data that speaker_play contains + memcpy(speakerdata0 + speaker_offset, payload, length); speaker_offset += length; + size_t bytes_written; + i2s_write(SPEAKER_I2S_NUMBER, speakerdata0, speaker_offset, &bytes_written, portMAX_DELAY); + speaker_offset = 0; // send data to server @@ -153,7 +180,7 @@ void websocket_setup() { Serial.println("connecting to WiFi"); } Serial.println("connected to WiFi"); - webSocket.begin("192.168.68.71", 9001, "/"); + webSocket.begin(COMPUTER_IP, 9001, "/"); webSocket.onEvent(webSocketEvent); // webSocket.setAuthorization("user", "Password"); webSocket.setReconnectInterval(5000); @@ -168,26 +195,43 @@ void setup() { delay(2000); } +bool recording = false; + +void flush_microphone() { + Serial.printf("[microphone] flushing %d bytes of data\n", data_offset); + if (data_offset == 0) return; + webSocket.sendBIN(microphonedata0, data_offset); + data_offset = 0; +} void loop() { - if (M5.Btn.isPressed()) { - data_offset = 0; + button.loop(); + if (button.justPressed()) { + Serial.println("Recording..."); + webSocket.sendTXT("s"); InitI2SSpeakerOrMic(MODE_MIC); - M5.dis.drawpix(0, CRGB(128, 128, 0)); - size_t byte_read; - - while (1) { - i2s_read(SPEAKER_I2S_NUMBER, - (char *)(microphonedata0 + data_offset), DATA_SIZE, - &byte_read, (100 / portTICK_RATE_MS)); - data_offset += 1024; - M5.update(); - if (M5.Btn.isReleased() || data_offset >= 71679) break; - // delay(60); - } - webSocket.sendBIN(microphonedata0, data_offset); + recording = true; + } else if (button.justReleased()) { + Serial.println("Stopped recording."); + webSocket.sendTXT("e"); + flush_microphone(); + recording = false; + } + if (recording) { + size_t bytes_read; + i2s_read( + SPEAKER_I2S_NUMBER, + (char *)(microphonedata0 + data_offset), + DATA_SIZE, &bytes_read, (100 / portTICK_RATE_MS) + ); + data_offset += bytes_read; + + if (data_offset > 1024*10) { + flush_microphone(); + } } + M5.update(); webSocket.loop(); diff --git a/01OS/01OS/clients/esp32/websocket_test.py b/01OS/01OS/clients/esp32/websocket_test.py index 71c6813..96afd54 100644 --- a/01OS/01OS/clients/esp32/websocket_test.py +++ b/01OS/01OS/clients/esp32/websocket_test.py @@ -1,33 +1,47 @@ #!/usr/bin/env python +"""A basic echo server for testing the device.""" + import asyncio -import simpleaudio as sa +import uuid +import websockets from websockets.server import serve +import traceback -def divide_chunks(l, n): - # looping till length l - for i in range(0, len(l), n): - yield l[i:i + n] - +def divide_chunks(l, n): + # looping till length l + for i in range(0, len(l), n): + yield l[i : i + n] -async def echo(websocket): - async for message in websocket: - try: - play_obj = sa.play_buffer(bytearray(message), 1, 2, 16000) - play_obj.wait_done() - x = list(divide_chunks(bytearray(message), 1000)) - for i in x: - await websocket.send(i) +buffers: dict[uuid.UUID, bytearray] = {} - except Exception as e: - print('fail') +async def echo(websocket: websockets.WebSocketServerProtocol): + async for message in websocket: + try: + if message == "s": + print("starting stream for", websocket.id) + buffers[websocket.id] = bytearray() + elif message == "e": + print("end, echoing stream for", websocket.id) + await websocket.send("s") + for chunk in divide_chunks(buffers[websocket.id], 1000): + await websocket.send(chunk) + await websocket.send("e") + elif type(message) is bytes: + print("recvd", len(message), "bytes from", websocket.id) + buffers[websocket.id].extend(message) + else: + print("ERR: recvd unknown message", message[:10], "from", websocket.id) + except Exception as _e: + traceback.print_exc() async def main(): async with serve(echo, "0.0.0.0", 9001): await asyncio.Future() # run forever -asyncio.run(main()) \ No newline at end of file + +asyncio.run(main()) From e156fec97843a18f3dd8fb996e62c3f037c79fd0 Mon Sep 17 00:00:00 2001 From: Shiven Mian Date: Sat, 17 Feb 2024 22:57:32 -0800 Subject: [PATCH 08/14] fix: have three btns for StepCheckDialog --- 01OS/01OS/server/teach.py | 51 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/01OS/01OS/server/teach.py b/01OS/01OS/server/teach.py index 78f7a88..1cd351f 100644 --- a/01OS/01OS/server/teach.py +++ b/01OS/01OS/server/teach.py @@ -1,9 +1,7 @@ from datetime import datetime from .utils.logs import setup_logging, logger -import tkinter as tk -import tkinter.simpledialog from interpreter import interpreter -from tkinter import messagebox +from tkinter import messagebox, Button, simpledialog, Tk, Label, Frame, LEFT, ACTIVE from ..utils.accumulator import Accumulator import time import os @@ -17,6 +15,39 @@ class Skill: self.steps = [] self.code = "" +class StepCheckDialog(simpledialog.Dialog): + def body(self, master): + self.title("Step Check") # Set the title of the dialog window + description = "Did I do this step correctly?" # Add window description + Label(master, text=description).pack() # Display window description + + def buttonbox(self): + box = Frame(self) + Button(box, text="Yes", width=10, command=self.yes_action, default=ACTIVE).pack(side=LEFT, padx=5, pady=5) + Button(box, text="No", width=10, command=self.no_action).pack(side=LEFT, padx=5, pady=5) + Button(box, text="Task Complete", width=10, command=self.task_complete_action).pack(side=LEFT, padx=5, pady=5) + + self.bind("", self.yes_action) + self.bind("", self.no_action) + + box.pack() + + def yes_action(self, event=None): + self.result = "Yes" + self.destroy() + + def no_action(self, event=None): + self.result = "No" + self.destroy() + + def task_complete_action(self, event=None): + self.result = "Task Complete" + self.destroy() + + def done(self, result): + self.result = result + self.destroy() + def to_camel_case(text): words = text.split() camel_case_string = words[0].lower() + ''.join(word.title() for word in words[1:]) @@ -36,13 +67,13 @@ def generate_python_steps(function_name, steps): return code_string def teach(): - root = tk.Tk() + root = Tk() root.withdraw() - skill_name = tkinter.simpledialog.askstring("Skill Name", "Please enter the name for the skill:") + skill_name = simpledialog.askstring("Skill Name", "Please enter the name for the skill:") skill = Skill(skill_name) while True: - step = tkinter.simpledialog.askstring("Next Step", "Enter the next step (or 'end' to finish): ") + step = simpledialog.askstring("Next Step", "Enter the next step (or 'end' to finish): ") logger.info(f"Performing step: {step}") if step == "end": break @@ -60,10 +91,14 @@ def teach(): time.sleep(0.05) accumulator.accumulate(chunk) - isCorrect = messagebox.askyesno("To Proceed?", "Did I do this step right?") - if isCorrect: + stepCheckDialog = StepCheckDialog(root) + stepCheckResult = stepCheckDialog.result + + if stepCheckResult == "Yes" or stepCheckResult == "Task Complete": skill.steps.append(step) skill.code += chunk_code + if stepCheckResult == "Task Complete": + break # Uncomment this incase you want steps instead of code #python_code = generate_python_steps(skill.skill_name, skill.steps) From c1688fb7ce4d5f0c6223b883ffdae46d0db44246 Mon Sep 17 00:00:00 2001 From: Shiven Mian Date: Sat, 17 Feb 2024 23:11:23 -0800 Subject: [PATCH 09/14] fix: small fixes --- 01OS/01OS/server/teach.py | 59 ++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/01OS/01OS/server/teach.py b/01OS/01OS/server/teach.py index 1cd351f..f08d9c0 100644 --- a/01OS/01OS/server/teach.py +++ b/01OS/01OS/server/teach.py @@ -69,36 +69,36 @@ def generate_python_steps(function_name, steps): def teach(): root = Tk() root.withdraw() - - skill_name = simpledialog.askstring("Skill Name", "Please enter the name for the skill:") - skill = Skill(skill_name) - while True: - step = simpledialog.askstring("Next Step", "Enter the next step (or 'end' to finish): ") - logger.info(f"Performing step: {step}") - if step == "end": - break - - chunk_code = "" - interpreter.computer.languages = [l for l in interpreter.computer.languages if l.name.lower() == "python"] - interpreter.force_task_completion = True - for chunk in interpreter.chat(step, stream=True, display=False): - if "format" in chunk and chunk["format"] == "execution": - content = chunk["content"] - language = content["format"] - code = content["content"] - chunk_code += code - interpreter.computer.run(code, language) - time.sleep(0.05) - accumulator.accumulate(chunk) - - stepCheckDialog = StepCheckDialog(root) - stepCheckResult = stepCheckDialog.result - - if stepCheckResult == "Yes" or stepCheckResult == "Task Complete": - skill.steps.append(step) - skill.code += chunk_code - if stepCheckResult == "Task Complete": + skill_name = simpledialog.askstring("Skill Name", "Please enter the name for the skill:", parent=root) + if skill_name: + skill = Skill(skill_name) + while True: + step = simpledialog.askstring("Next Step", "Enter the next step (or 'end' to finish): ", parent=root) + if step is None or step == "end": break + elif step.strip() == "": + continue + logger.info(f"Performing step: {step}") + chunk_code = "" + interpreter.computer.languages = [l for l in interpreter.computer.languages if l.name.lower() == "python"] + interpreter.force_task_completion = True + for chunk in interpreter.chat(step, stream=True, display=False): + if "format" in chunk and chunk["format"] == "execution": + content = chunk["content"] + language = content["format"] + code = content["content"] + chunk_code += code + interpreter.computer.run(code, language) + time.sleep(0.05) + accumulator.accumulate(chunk) + + stepCheckDialog = StepCheckDialog(root) + stepCheckResult = stepCheckDialog.result + if stepCheckResult == "Yes" or stepCheckResult == "Task Complete": + skill.steps.append(step) + skill.code += chunk_code + if stepCheckResult == "Task Complete": + break # Uncomment this incase you want steps instead of code #python_code = generate_python_steps(skill.skill_name, skill.steps) @@ -106,5 +106,6 @@ def teach(): python_code = generate_python_code(skill.skill_name, skill.code) SKILLS_DIR = os.path.dirname(__file__) + "/skills" filename = os.path.join(SKILLS_DIR, f"{skill.skill_name.replace(' ', '_')}.py") + logger.info(f"Saving skill to: {filename}") with open(filename, "w") as file: file.write(python_code) From 968aa854c0ceffe995ed8755384dd2c61a138b23 Mon Sep 17 00:00:00 2001 From: Theia Vogel Date: Sun, 18 Feb 2024 01:30:30 -0800 Subject: [PATCH 10/14] conform device protocol to LMC (w/ hacks) --- 01OS/01OS/clients/esp32/playback/playback.ino | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/01OS/01OS/clients/esp32/playback/playback.ino b/01OS/01OS/clients/esp32/playback/playback.ino index 7f2e2a3..d4cc29a 100644 --- a/01OS/01OS/clients/esp32/playback/playback.ino +++ b/01OS/01OS/clients/esp32/playback/playback.ino @@ -70,6 +70,7 @@ void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) { } void InitI2SSpeakerOrMic(int mode) { + Serial.printf("InitI2sSpeakerOrMic %d\n", mode); esp_err_t err = ESP_OK; i2s_driver_uninstall(SPEAKER_I2S_NUMBER); @@ -136,17 +137,19 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { break; case WStype_TEXT: Serial.printf("[WSc] get text: %s\n", payload); - if ((char)payload[0] == 's'){ - Serial.println("start"); - speaker_offset = 0; - InitI2SSpeakerOrMic(MODE_SPK); + { + std::string str(payload, payload + length); + bool isAudio = str.find("\"audio\"") != std::string::npos; + if (isAudio && str.find("\"start\"") != std::string::npos) { + Serial.println("start playback"); + speaker_offset = 0; + InitI2SSpeakerOrMic(MODE_SPK); + } else if (isAudio && str.find("\"end\"") != std::string::npos) { + Serial.println("end playback"); + // speaker_play(speakerdata0, speaker_offset); + // speaker_offset = 0; + } } - if ((char)payload[0] == 'e'){ - Serial.println("end"); - // speaker_play(speakerdata0, speaker_offset); - // speaker_offset = 0; - } - // send message to server // webSocket.sendTXT("message here"); break; @@ -180,12 +183,12 @@ void websocket_setup() { Serial.println("connecting to WiFi"); } Serial.println("connected to WiFi"); - webSocket.begin(COMPUTER_IP, 9001, "/"); + webSocket.begin(COMPUTER_IP, 8000, "/"); webSocket.onEvent(webSocketEvent); // webSocket.setAuthorization("user", "Password"); webSocket.setReconnectInterval(5000); - } + void setup() { M5.begin(true, false, true); M5.dis.drawpix(0, CRGB(128, 128, 0)); @@ -208,17 +211,19 @@ void loop() { button.loop(); if (button.justPressed()) { Serial.println("Recording..."); - webSocket.sendTXT("s"); + webSocket.sendTXT("{\"role\": \"user\", \"type\": \"audio\", \"format\": \"bytes.raw\", \"start\": true}"); InitI2SSpeakerOrMic(MODE_MIC); recording = true; + data_offset = 0; + Serial.println("Recording ready."); } else if (button.justReleased()) { Serial.println("Stopped recording."); - webSocket.sendTXT("e"); + webSocket.sendTXT("{\"role\": \"user\", \"type\": \"audio\", \"format\": \"bytes.raw\", \"end\": true}"); flush_microphone(); recording = false; - } - - if (recording) { + data_offset = 0; + } else if (recording) { + Serial.printf("Reading chunk at %d...\n", data_offset); size_t bytes_read; i2s_read( SPEAKER_I2S_NUMBER, @@ -226,13 +231,13 @@ void loop() { DATA_SIZE, &bytes_read, (100 / portTICK_RATE_MS) ); data_offset += bytes_read; + Serial.printf("Read %d bytes in chunk.\n", bytes_read); - if (data_offset > 1024*10) { + if (data_offset > 1024*9) { flush_microphone(); } } M5.update(); webSocket.loop(); - } From c031637fff230135dfaf1f088c721fa9ae7cd064 Mon Sep 17 00:00:00 2001 From: Theia Vogel Date: Sun, 18 Feb 2024 01:30:46 -0800 Subject: [PATCH 11/14] system msg / function calling tweaks --- 01OS/01OS/server/i.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/01OS/01OS/server/i.py b/01OS/01OS/server/i.py index 3a68101..2ab2bb2 100644 --- a/01OS/01OS/server/i.py +++ b/01OS/01OS/server/i.py @@ -43,6 +43,18 @@ def configure_interpreter(interpreter: OpenInterpreter): Remember: You can run Python code. Be very concise. Ensure that you actually run code every time! THIS IS IMPORTANT. You NEED to write code. **Help the user by being very concise in your answers.** Do not break down tasks excessively, just into simple, few minute steps. Don't assume the user lives their life in a certain way— pick very general tasks if you're breaking a task down. + ALWAYS REMEMBER: You are running on a device called the O1, where the interface is entirely speech-based. Keep your responses succint in light of this! + IF YOU NEED TO THINK ABOUT A PROBLEM: (such as "Here's the plan:"), WRITE IT IN THE COMMENTS of the code block! + For example: + > User: What is 432/7? + > Assistant: Let me use Python to calculate that. + > Assistant Python function call: + > # Here's the plan: + > # 1. Divide the numbers + > # 2. Round it to 3 digits. + > print(round(432/7, 3)) + > Assistant: 432 / 7 is 61.714. + Use the following functions (assume they're imported) to complete your goals whenever possible: {{ import sys @@ -70,7 +82,9 @@ print(output) """.strip() - interpreter.custom_instructions = system_message + # interpreter.custom_instructions = system_message + interpreter.system_message = system_message + interpreter.llm.supports_functions = True ### LLM SETTINGS From c4fa3db52bb649278380691a6d6aa4dd1b7e3c2e Mon Sep 17 00:00:00 2001 From: Theia Vogel Date: Sun, 18 Feb 2024 01:30:56 -0800 Subject: [PATCH 12/14] bug whacking --- 01OS/01OS/server/server.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/01OS/01OS/server/server.py b/01OS/01OS/server/server.py index 553ee27..d05ee6c 100644 --- a/01OS/01OS/server/server.py +++ b/01OS/01OS/server/server.py @@ -201,7 +201,7 @@ async def listener(): accumulated_text = "" - for chunk in interpreter.chat(messages, stream=True, display=False): + for chunk in interpreter.chat(messages, stream=True, display=True): logger.debug("Got chunk:", chunk) @@ -212,7 +212,7 @@ async def listener(): if os.getenv('TTS_RUNNER') == "server": # Speak full sentences out loud - if chunk["role"] == "assistant" and "content" in chunk: + if chunk["role"] == "assistant" and "content" in chunk and chunk["type"] == "message": accumulated_text += chunk["content"] sentences = split_into_sentences(accumulated_text) @@ -241,7 +241,7 @@ async def listener(): # Check if it's just an end flag. We ignore those. temp_message = await from_user.get() - if temp_message == {'role': 'user', 'type': 'message', 'end': True}: + if type(temp_message) is dict and temp_message.get("role") == "user" and temp_message.get("end"): # Yup. False alarm. continue else: @@ -251,8 +251,9 @@ async def listener(): with open(conversation_history_path, 'w') as file: json.dump(interpreter.messages, file, indent=4) - logger.info("New user message recieved. Breaking.") - break + # TODO: is triggering seemingly randomly + #logger.info("New user message recieved. Breaking.") + #break # Also check if there's any new computer messages if not from_computer.empty(): From 801a049b77bdb2d983acf9a8b2c56be866168d09 Mon Sep 17 00:00:00 2001 From: Theia Vogel Date: Sun, 18 Feb 2024 01:31:07 -0800 Subject: [PATCH 13/14] support audio/raw in stt --- 01OS/01OS/server/stt/stt.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/01OS/01OS/server/stt/stt.py b/01OS/01OS/server/stt/stt.py index 2a57c9b..588e9e4 100644 --- a/01OS/01OS/server/stt/stt.py +++ b/01OS/01OS/server/stt/stt.py @@ -25,6 +25,8 @@ def convert_mime_type_to_format(mime_type: str) -> str: return "wav" if mime_type == "audio/webm": return "webm" + if mime_type == "audio/raw": + return "dat" return mime_type @@ -43,7 +45,16 @@ def export_audio_to_wav_ffmpeg(audio: bytearray, mime_type: str) -> str: # Export to wav output_path = os.path.join(temp_dir, f"output_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav") - ffmpeg.input(input_path).output(output_path, acodec='pcm_s16le', ac=1, ar='16k').run() + print(mime_type, input_path, output_path) + if mime_type == "audio/raw": + ffmpeg.input( + input_path, + f='s16le', + ar='16000', + ac=1, + ).output(output_path).run() + else: + ffmpeg.input(input_path).output(output_path, acodec='pcm_s16le', ac=1, ar='16k').run() try: yield output_path From 5348f569b0ca18d4bfc790d1030d6d82caee7ca9 Mon Sep 17 00:00:00 2001 From: Theia Vogel Date: Sun, 18 Feb 2024 01:31:16 -0800 Subject: [PATCH 14/14] quick hack to support audio/raw in tts --- 01OS/01OS/server/tts/tts.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/01OS/01OS/server/tts/tts.py b/01OS/01OS/server/tts/tts.py index 6106966..ec16cc0 100644 --- a/01OS/01OS/server/tts/tts.py +++ b/01OS/01OS/server/tts/tts.py @@ -6,6 +6,7 @@ from pydub import AudioSegment from dotenv import load_dotenv load_dotenv() # take environment variables from .env. +import ffmpeg import tempfile from openai import OpenAI import os @@ -28,11 +29,17 @@ def stream_tts(text): input=text, response_format="opus" ) - with tempfile.NamedTemporaryFile(suffix=".opus") as temp_file: + with tempfile.NamedTemporaryFile(suffix=".opus", delete=False) as temp_file: response.stream_to_file(temp_file.name) - audio_bytes = temp_file.read() - file_type = "bytes.opus" + # TODO: hack to format audio correctly for device + outfile = tempfile.gettempdir() + "/" + "raw.dat" + ffmpeg.input(temp_file.name).output(outfile, f="s16le", ar="16000", ac="1").run() + with open(outfile, "rb") as f: + audio_bytes = f.read() + file_type = "bytes.raw" + print(outfile, len(audio_bytes)) + os.remove(outfile) else: with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file: