From 1a69ce2d9723fbf3e6e73c46ad2779ad3f7736c0 Mon Sep 17 00:00:00 2001 From: Totoo Date: Fri, 5 Jan 2024 07:43:30 +0100 Subject: [PATCH] Accessibility over serial (#1717) * Initial accessibility support * added it to some widgets to test * More widget accessibility response * More widgets, better output * Mark selected widget on list * typo --- firmware/application/event_m0.cpp | 8 ++ firmware/application/event_m0.hpp | 3 + firmware/application/ui/ui_receiver.cpp | 7 ++ firmware/application/ui/ui_receiver.hpp | 3 + firmware/application/usb_serial_shell.cpp | 84 ++++++++++++++ firmware/common/ui_widget.cpp | 127 ++++++++++++++++++++++ firmware/common/ui_widget.hpp | 43 ++++++++ 7 files changed, 275 insertions(+) diff --git a/firmware/application/event_m0.cpp b/firmware/application/event_m0.cpp index a05c59a3..65ce0a57 100644 --- a/firmware/application/event_m0.cpp +++ b/firmware/application/event_m0.cpp @@ -274,6 +274,14 @@ void EventDispatcher::on_touch_event(ui::TouchEvent event) { } } +ui::Widget* EventDispatcher::getTopWidget() { + return top_widget; +} + +ui::Widget* EventDispatcher::getFocusedWidget() { + return context.focus_manager().focus_widget(); +} + void EventDispatcher::handle_lcd_frame_sync() { DisplayFrameSyncMessage message; message_map.send(&message); diff --git a/firmware/application/event_m0.hpp b/firmware/application/event_m0.hpp index 1bf705ee..c4fbbdb1 100644 --- a/firmware/application/event_m0.hpp +++ b/firmware/application/event_m0.hpp @@ -89,6 +89,9 @@ class EventDispatcher { void emulateTouch(ui::TouchEvent event); void emulateKeyboard(ui::KeyboardEvent event); + ui::Widget* getTopWidget(); + ui::Widget* getFocusedWidget(); + private: static Thread* thread_event_loop; diff --git a/firmware/application/ui/ui_receiver.cpp b/firmware/application/ui/ui_receiver.cpp index e79e2abd..283e849f 100644 --- a/firmware/application/ui/ui_receiver.cpp +++ b/firmware/application/ui/ui_receiver.cpp @@ -93,6 +93,13 @@ void FrequencyField::set_allow_digit_mode(bool allowed) { } } +void FrequencyField::getAccessibilityText(std::string& result) { + result = to_string_dec_int(value_); +} +void FrequencyField::getWidgetName(std::string& result) { + result = "FrequencyField"; +} + void FrequencyField::paint(Painter& painter) { const auto str_value = to_string_short_freq(value_); const auto paint_style = has_focus() ? style().invert() : style(); diff --git a/firmware/application/ui/ui_receiver.hpp b/firmware/application/ui/ui_receiver.hpp index 4652bc5b..2b702dd9 100644 --- a/firmware/application/ui/ui_receiver.hpp +++ b/firmware/application/ui/ui_receiver.hpp @@ -66,6 +66,9 @@ class FrequencyField : public Widget { void on_focus() override; void on_blur() override; + void getAccessibilityText(std::string& result) override; + void getWidgetName(std::string& result) override; + private: const size_t length_; range_t range_; diff --git a/firmware/application/usb_serial_shell.cpp b/firmware/application/usb_serial_shell.cpp index a3267611..64435990 100644 --- a/firmware/application/usb_serial_shell.cpp +++ b/firmware/application/usb_serial_shell.cpp @@ -41,6 +41,7 @@ #include "chprintf.h" #include "chqueues.h" #include "untar.hpp" +#include "ui_widget.hpp" #include #include @@ -730,6 +731,87 @@ static void cpld_info(BaseSequentialStream* chp, int argc, char* argv[]) { } } +// walks throught the given widget's childs in recurse to get all support text and pass it to a callback function +static void widget_collect_accessibility(BaseSequentialStream* chp, ui::Widget* w, void (*callback)(BaseSequentialStream*, const std::string&, const std::string&), ui::Widget* focusedWidget) { + for (auto child : w->children()) { + if (!child->hidden()) { + std::string res = ""; + child->getAccessibilityText(res); + std::string strtype = ""; + child->getWidgetName(strtype); + if (child == focusedWidget) strtype += "*"; + if (callback != NULL && !res.empty()) callback(chp, res, strtype); + widget_collect_accessibility(chp, child, callback, focusedWidget); + } + } +} + +// callback when it found any response from a widget +static void accessibility_callback(BaseSequentialStream* chp, const std::string& strResult, const std::string& wgType) { + if (!wgType.empty()) { + chprintf(chp, "["); + chprintf(chp, wgType.c_str()); + chprintf(chp, "] "); + } + chprintf(chp, "%s\r\n", strResult.c_str()); +} + +// gets all widget's accessibility helper text +static void cmd_accessibility_readall(BaseSequentialStream* chp, int argc, char* argv[]) { + (void)argc; + (void)argv; + + auto evtd = getEventDispatcherInstance(); + if (evtd == NULL) { + chprintf(chp, "error Can't get Event Dispatcherr\n"); + return; + } + auto wg = evtd->getTopWidget(); + if (wg == NULL) { + chprintf(chp, "error Can't get top Widget\r\n"); + return; + } + auto focused = evtd->getFocusedWidget(); + widget_collect_accessibility(chp, wg, accessibility_callback, focused); + chprintf(chp, "ok\r\n"); +} + +// gets focused widget's accessibility helper text +static void cmd_accessibility_readcurr(BaseSequentialStream* chp, int argc, char* argv[]) { + (void)argc; + (void)argv; + + auto evtd = getEventDispatcherInstance(); + if (evtd == NULL) { + chprintf(chp, "error Can't get Event Dispatcher\r\n"); + return; + } + auto wg = evtd->getFocusedWidget(); + if (wg == NULL) { + chprintf(chp, "error Can't get focused Widget\r\n"); + return; + } + std::string res = ""; + wg->getAccessibilityText(res); + if (res.empty()) { + // try with parent + wg = wg->parent(); + if (wg == NULL) { + chprintf(chp, "error Widget not providing accessibility info\r\n"); + return; + } + wg->getAccessibilityText(res); + if (res.empty()) { + chprintf(chp, "error Widget not providing accessibility info\r\n"); + return; + } + } + std::string strtype = ""; + wg->getWidgetName(strtype); + accessibility_callback(chp, res, strtype); + chprintf(chp, "\r\nok\r\n"); +} + static void cmd_cpld_read(BaseSequentialStream* chp, int argc, char* argv[]) { const char* usage = "usage: cpld_read \r\n" @@ -915,6 +997,8 @@ static const ShellCommand commands[] = { {"filesize", cmd_sd_filesize}, {"cpld_info", cpld_info}, {"cpld_read", cmd_cpld_read}, + {"accessibility_readall", cmd_accessibility_readall}, + {"accessibility_readcurr", cmd_accessibility_readcurr}, {NULL, NULL}}; static const ShellConfig shell_cfg1 = { diff --git a/firmware/common/ui_widget.cpp b/firmware/common/ui_widget.cpp index 2b42b19a..4860173f 100644 --- a/firmware/common/ui_widget.cpp +++ b/firmware/common/ui_widget.cpp @@ -237,6 +237,12 @@ void Widget::dirty_overlapping_children_in_rect(const Rect& child_rect) { } } +void Widget::getAccessibilityText(std::string& result) { + result = ""; +} +void Widget::getWidgetName(std::string& result) { + result = ""; +} /* View ******************************************************************/ void View::paint(Painter& painter) { @@ -367,6 +373,12 @@ void Text::set(std::string_view value) { set_dirty(); } +void Text::getAccessibilityText(std::string& result) { + result = text; +} +void Text::getWidgetName(std::string& result) { + result = "Text"; +} void Text::paint(Painter& painter) { const auto rect = screen_rect(); auto s = has_focus() ? style().invert() : style(); @@ -407,6 +419,17 @@ void Labels::paint(Painter& painter) { } } +void Labels::getAccessibilityText(std::string& result) { + result = ""; + for (auto& label : labels_) { + result += label.text; + result += ", "; + } +} +void Labels::getWidgetName(std::string& result) { + result = "Labels"; +} + /* LiveDateTime **********************************************************/ void LiveDateTime::on_tick_second() { @@ -579,6 +602,13 @@ void ProgressBar::set_value(const uint32_t value) { set_dirty(); } +void ProgressBar::getAccessibilityText(std::string& result) { + result = to_string_dec_uint(_value) + " / " + to_string_dec_uint(_max); +} +void ProgressBar::getWidgetName(std::string& result) { + result = "ProgressBar"; +} + void ProgressBar::paint(Painter& painter) { int v_scaled; @@ -679,6 +709,13 @@ void Console::write(std::string message) { } } +void Console::getAccessibilityText(std::string& result) { + result = "{" + buffer + "}"; +} +void Console::getWidgetName(std::string& result) { + result = "Console"; +} + void Console::writeln(std::string message) { write(message + "\n"); } @@ -787,6 +824,13 @@ bool Checkbox::set_value(const bool value) { return false; } +void Checkbox::getAccessibilityText(std::string& result) { + result = text_ + ((value_) ? " checked" : " unchecked"); +} +void Checkbox::getWidgetName(std::string& result) { + result = "Checkbox"; +} + bool Checkbox::value() const { return value_; } @@ -904,6 +948,13 @@ std::string Button::text() const { return text_; } +void Button::getAccessibilityText(std::string& result) { + result = text_; +} +void Button::getWidgetName(std::string& result) { + result = "Button"; +} + void Button::paint(Painter& painter) { Color bg, fg; const auto r = screen_rect(); @@ -1050,6 +1101,13 @@ std::string ButtonWithEncoder::text() const { return text_; } +void ButtonWithEncoder::getAccessibilityText(std::string& result) { + result = text_; +} +void ButtonWithEncoder::getWidgetName(std::string& result) { + result = "ButtonWithEncoder"; +} + void ButtonWithEncoder::paint(Painter& painter) { Color bg, fg; const auto r = screen_rect(); @@ -1205,6 +1263,13 @@ void NewButton::set_text(const std::string value) { set_dirty(); } +void NewButton::getAccessibilityText(std::string& result) { + result = text_; +} +void NewButton::getWidgetName(std::string& result) { + result = "NewButton"; +} + std::string NewButton::text() const { return text_; } @@ -1399,6 +1464,13 @@ ImageButton::ImageButton( set_focusable(true); } +void ImageButton::getAccessibilityText(std::string& result) { + result = "image"; +} +void ImageButton::getWidgetName(std::string& result) { + result = "ImageButton"; +} + bool ImageButton::on_key(const KeyEvent key) { if (key == KeyEvent::Select) { if (on_select) { @@ -1489,6 +1561,13 @@ bool ImageToggle::value() const { return value_; } +void ImageToggle::getAccessibilityText(std::string& result) { + result = value_ ? "checked" : "unchecked"; +} +void ImageToggle::getWidgetName(std::string& result) { + result = "ImageToggle"; +} + void ImageToggle::set_value(bool b) { if (b == value_) return; @@ -1524,6 +1603,13 @@ size_t ImageOptionsField::selected_index_value() const { return options[selected_index_].second; } +void ImageOptionsField::getAccessibilityText(std::string& result) { + result = "selected index: " + to_string_dec_uint(selected_index_); +} +void ImageOptionsField::getWidgetName(std::string& result) { + result = "ImageOptionsField"; +} + void ImageOptionsField::set_selected_index(const size_t new_index) { if (new_index < options.size()) { if (new_index != selected_index()) { @@ -1623,6 +1709,20 @@ const OptionsField::value_t& OptionsField::selected_index_value() const { return options_[selected_index_].second; } +void OptionsField::getAccessibilityText(std::string& result) { + result = "options: "; + bool first = true; + for (const auto& option : options_) { + if (!first) result += " ,"; + first = false; + result += option.first; + } + result += "; selected: " + selected_index_name(); +} +void OptionsField::getWidgetName(std::string& result) { + result = "OptionsField"; +} + void OptionsField::set_selected_index(const size_t new_index, bool trigger_change) { if (new_index < options_.size()) { if (new_index != selected_index() || trigger_change) { @@ -1741,6 +1841,13 @@ const std::string& TextEdit::value() const { return text_; } +void TextEdit::getAccessibilityText(std::string& result) { + result = text_; +} +void TextEdit::getWidgetName(std::string& result) { + result = "TextEdit"; +} + void TextEdit::set_cursor(uint32_t pos) { cursor_pos_ = std::min(pos, text_.length()); set_dirty(); @@ -1890,6 +1997,13 @@ const std::string& TextField::get_text() const { return text; } +void TextField::getAccessibilityText(std::string& result) { + result = text; +} +void TextField::getWidgetName(std::string& result) { + result = "TextField"; +} + void TextField::set_text(std::string_view value) { set(value); if (on_change) @@ -1945,6 +2059,13 @@ int32_t NumberField::value() const { return value_; } +void NumberField::getAccessibilityText(std::string& result) { + result = to_string_dec_int(value_); +} +void NumberField::getWidgetName(std::string& result) { + result = "NumberField"; +} + void NumberField::set_value(int32_t new_value, bool trigger_change) { if (can_loop) { if (new_value >= range.first) @@ -2155,6 +2276,12 @@ const std::string& SymField::to_string() const { return value_; } +void SymField::getAccessibilityText(std::string& result) { + result = value_; +} +void SymField::getWidgetName(std::string& result) { + result = "SymField"; +} void SymField::paint(Painter& painter) { Point p = screen_pos(); diff --git a/firmware/common/ui_widget.hpp b/firmware/common/ui_widget.hpp index ff80bc16..0c989234 100644 --- a/firmware/common/ui_widget.hpp +++ b/firmware/common/ui_widget.hpp @@ -106,6 +106,9 @@ class Widget { virtual Context& context() const; + virtual void getAccessibilityText(std::string& result); + virtual void getWidgetName(std::string& result); + void set_style(const Style* new_style); const Style& style() const; @@ -207,6 +210,8 @@ class Text : public Widget { void set(std::string_view value); void paint(Painter& painter) override; + void getAccessibilityText(std::string& result) override; + void getWidgetName(std::string& result) override; protected: // NB: Don't truncate this string. The UI will only render @@ -234,6 +239,8 @@ class Labels : public Widget { void set_labels(std::initializer_list