Accessibility over serial (#1717)

* Initial accessibility support
* added it to some widgets to test
* More widget accessibility response
* More widgets, better output
* Mark selected widget on list
* typo

firmware/application/event_m0.cpp

@@ -274,6 +274,14 @@ void EventDispatcher::on_touch_event(ui::TouchEvent event) {
     }
 }
 
+ui::Widget* EventDispatcher::getTopWidget() {
+    return top_widget;
+}
+
+ui::Widget* EventDispatcher::getFocusedWidget() {
+    return context.focus_manager().focus_widget();
+}
+
 void EventDispatcher::handle_lcd_frame_sync() {
     DisplayFrameSyncMessage message;
     message_map.send(&message);

firmware/application/event_m0.hpp

@@ -89,6 +89,9 @@ class EventDispatcher {
     void emulateTouch(ui::TouchEvent event);
     void emulateKeyboard(ui::KeyboardEvent event);
 
+    ui::Widget* getTopWidget();
+    ui::Widget* getFocusedWidget();
+
    private:
     static Thread* thread_event_loop;
 

firmware/application/ui/ui_receiver.cpp

@@ -93,6 +93,13 @@ void FrequencyField::set_allow_digit_mode(bool allowed) {
     }
 }
 
+void FrequencyField::getAccessibilityText(std::string& result) {
+    result = to_string_dec_int(value_);
+}
+void FrequencyField::getWidgetName(std::string& result) {
+    result = "FrequencyField";
+}
+
 void FrequencyField::paint(Painter& painter) {
     const auto str_value = to_string_short_freq(value_);
     const auto paint_style = has_focus() ? style().invert() : style();

firmware/application/ui/ui_receiver.hpp

@@ -66,6 +66,9 @@ class FrequencyField : public Widget {
     void on_focus() override;
     void on_blur() override;
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
    private:
     const size_t length_;
     range_t range_;

firmware/application/usb_serial_shell.cpp

@@ -41,6 +41,7 @@
 #include "chprintf.h"
 #include "chqueues.h"
 #include "untar.hpp"
+#include "ui_widget.hpp"
 
 #include <string>
 #include <codecvt>
@@ -730,6 +731,87 @@ static void cpld_info(BaseSequentialStream* chp, int argc, char* argv[]) {
     }
 }
 
+// walks throught the given widget's childs in recurse to get all support text and pass it to a callback function
+static void widget_collect_accessibility(BaseSequentialStream* chp, ui::Widget* w, void (*callback)(BaseSequentialStream*, const std::string&, const std::string&), ui::Widget* focusedWidget) {
+    for (auto child : w->children()) {
+        if (!child->hidden()) {
+            std::string res = "";
+            child->getAccessibilityText(res);
+            std::string strtype = "";
+            child->getWidgetName(strtype);
+            if (child == focusedWidget) strtype += "*";
+            if (callback != NULL && !res.empty()) callback(chp, res, strtype);
+            widget_collect_accessibility(chp, child, callback, focusedWidget);
+        }
+    }
+}
+
+// callback when it found any response from a widget
+static void accessibility_callback(BaseSequentialStream* chp, const std::string& strResult, const std::string& wgType) {
+    if (!wgType.empty()) {
+        chprintf(chp, "[");
+        chprintf(chp, wgType.c_str());
+        chprintf(chp, "] ");
+    }
+    chprintf(chp, "%s\r\n", strResult.c_str());
+}
+
+// gets all widget's accessibility helper text
+static void cmd_accessibility_readall(BaseSequentialStream* chp, int argc, char* argv[]) {
+    (void)argc;
+    (void)argv;
+
+    auto evtd = getEventDispatcherInstance();
+    if (evtd == NULL) {
+        chprintf(chp, "error Can't get Event Dispatcherr\n");
+        return;
+    }
+    auto wg = evtd->getTopWidget();
+    if (wg == NULL) {
+        chprintf(chp, "error Can't get top Widget\r\n");
+        return;
+    }
+    auto focused = evtd->getFocusedWidget();
+    widget_collect_accessibility(chp, wg, accessibility_callback, focused);
+    chprintf(chp, "ok\r\n");
+}
+
+// gets focused widget's accessibility helper text
+static void cmd_accessibility_readcurr(BaseSequentialStream* chp, int argc, char* argv[]) {
+    (void)argc;
+    (void)argv;
+
+    auto evtd = getEventDispatcherInstance();
+    if (evtd == NULL) {
+        chprintf(chp, "error Can't get Event Dispatcher\r\n");
+        return;
+    }
+    auto wg = evtd->getFocusedWidget();
+    if (wg == NULL) {
+        chprintf(chp, "error Can't get focused Widget\r\n");
+        return;
+    }
+    std::string res = "";
+    wg->getAccessibilityText(res);
+    if (res.empty()) {
+        // try with parent
+        wg = wg->parent();
+        if (wg == NULL) {
+            chprintf(chp, "error Widget not providing accessibility info\r\n");
+            return;
+        }
+        wg->getAccessibilityText(res);
+        if (res.empty()) {
+            chprintf(chp, "error Widget not providing accessibility info\r\n");
+            return;
+        }
+    }
+    std::string strtype = "";
+    wg->getWidgetName(strtype);
+    accessibility_callback(chp, res, strtype);
+    chprintf(chp, "\r\nok\r\n");
+}
+
 static void cmd_cpld_read(BaseSequentialStream* chp, int argc, char* argv[]) {
     const char* usage =
         "usage: cpld_read <device> <target>\r\n"
@@ -915,6 +997,8 @@ static const ShellCommand commands[] = {
     {"filesize", cmd_sd_filesize},
     {"cpld_info", cpld_info},
     {"cpld_read", cmd_cpld_read},
+    {"accessibility_readall", cmd_accessibility_readall},
+    {"accessibility_readcurr", cmd_accessibility_readcurr},
     {NULL, NULL}};
 
 static const ShellConfig shell_cfg1 = {

firmware/common/ui_widget.cpp

@@ -237,6 +237,12 @@ void Widget::dirty_overlapping_children_in_rect(const Rect& child_rect) {
     }
 }
 
+void Widget::getAccessibilityText(std::string& result) {
+    result = "";
+}
+void Widget::getWidgetName(std::string& result) {
+    result = "";
+}
 /* View ******************************************************************/
 
 void View::paint(Painter& painter) {
@@ -367,6 +373,12 @@ void Text::set(std::string_view value) {
     set_dirty();
 }
 
+void Text::getAccessibilityText(std::string& result) {
+    result = text;
+}
+void Text::getWidgetName(std::string& result) {
+    result = "Text";
+}
 void Text::paint(Painter& painter) {
     const auto rect = screen_rect();
     auto s = has_focus() ? style().invert() : style();
@@ -407,6 +419,17 @@ void Labels::paint(Painter& painter) {
     }
 }
 
+void Labels::getAccessibilityText(std::string& result) {
+    result = "";
+    for (auto& label : labels_) {
+        result += label.text;
+        result += ", ";
+    }
+}
+void Labels::getWidgetName(std::string& result) {
+    result = "Labels";
+}
+
 /* LiveDateTime **********************************************************/
 
 void LiveDateTime::on_tick_second() {
@@ -579,6 +602,13 @@ void ProgressBar::set_value(const uint32_t value) {
     set_dirty();
 }
 
+void ProgressBar::getAccessibilityText(std::string& result) {
+    result = to_string_dec_uint(_value) + " / " + to_string_dec_uint(_max);
+}
+void ProgressBar::getWidgetName(std::string& result) {
+    result = "ProgressBar";
+}
+
 void ProgressBar::paint(Painter& painter) {
     int v_scaled;
 
@@ -679,6 +709,13 @@ void Console::write(std::string message) {
     }
 }
 
+void Console::getAccessibilityText(std::string& result) {
+    result = "{" + buffer + "}";
+}
+void Console::getWidgetName(std::string& result) {
+    result = "Console";
+}
+
 void Console::writeln(std::string message) {
     write(message + "\n");
 }
@@ -787,6 +824,13 @@ bool Checkbox::set_value(const bool value) {
     return false;
 }
 
+void Checkbox::getAccessibilityText(std::string& result) {
+    result = text_ + ((value_) ? " checked" : " unchecked");
+}
+void Checkbox::getWidgetName(std::string& result) {
+    result = "Checkbox";
+}
+
 bool Checkbox::value() const {
     return value_;
 }
@@ -904,6 +948,13 @@ std::string Button::text() const {
     return text_;
 }
 
+void Button::getAccessibilityText(std::string& result) {
+    result = text_;
+}
+void Button::getWidgetName(std::string& result) {
+    result = "Button";
+}
+
 void Button::paint(Painter& painter) {
     Color bg, fg;
     const auto r = screen_rect();
@@ -1050,6 +1101,13 @@ std::string ButtonWithEncoder::text() const {
     return text_;
 }
 
+void ButtonWithEncoder::getAccessibilityText(std::string& result) {
+    result = text_;
+}
+void ButtonWithEncoder::getWidgetName(std::string& result) {
+    result = "ButtonWithEncoder";
+}
+
 void ButtonWithEncoder::paint(Painter& painter) {
     Color bg, fg;
     const auto r = screen_rect();
@@ -1205,6 +1263,13 @@ void NewButton::set_text(const std::string value) {
     set_dirty();
 }
 
+void NewButton::getAccessibilityText(std::string& result) {
+    result = text_;
+}
+void NewButton::getWidgetName(std::string& result) {
+    result = "NewButton";
+}
+
 std::string NewButton::text() const {
     return text_;
 }
@@ -1399,6 +1464,13 @@ ImageButton::ImageButton(
     set_focusable(true);
 }
 
+void ImageButton::getAccessibilityText(std::string& result) {
+    result = "image";
+}
+void ImageButton::getWidgetName(std::string& result) {
+    result = "ImageButton";
+}
+
 bool ImageButton::on_key(const KeyEvent key) {
     if (key == KeyEvent::Select) {
         if (on_select) {
@@ -1489,6 +1561,13 @@ bool ImageToggle::value() const {
     return value_;
 }
 
+void ImageToggle::getAccessibilityText(std::string& result) {
+    result = value_ ? "checked" : "unchecked";
+}
+void ImageToggle::getWidgetName(std::string& result) {
+    result = "ImageToggle";
+}
+
 void ImageToggle::set_value(bool b) {
     if (b == value_)
         return;
@@ -1524,6 +1603,13 @@ size_t ImageOptionsField::selected_index_value() const {
     return options[selected_index_].second;
 }
 
+void ImageOptionsField::getAccessibilityText(std::string& result) {
+    result = "selected index: " + to_string_dec_uint(selected_index_);
+}
+void ImageOptionsField::getWidgetName(std::string& result) {
+    result = "ImageOptionsField";
+}
+
 void ImageOptionsField::set_selected_index(const size_t new_index) {
     if (new_index < options.size()) {
         if (new_index != selected_index()) {
@@ -1623,6 +1709,20 @@ const OptionsField::value_t& OptionsField::selected_index_value() const {
     return options_[selected_index_].second;
 }
 
+void OptionsField::getAccessibilityText(std::string& result) {
+    result = "options: ";
+    bool first = true;
+    for (const auto& option : options_) {
+        if (!first) result += " ,";
+        first = false;
+        result += option.first;
+    }
+    result += "; selected: " + selected_index_name();
+}
+void OptionsField::getWidgetName(std::string& result) {
+    result = "OptionsField";
+}
+
 void OptionsField::set_selected_index(const size_t new_index, bool trigger_change) {
     if (new_index < options_.size()) {
         if (new_index != selected_index() || trigger_change) {
@@ -1741,6 +1841,13 @@ const std::string& TextEdit::value() const {
     return text_;
 }
 
+void TextEdit::getAccessibilityText(std::string& result) {
+    result = text_;
+}
+void TextEdit::getWidgetName(std::string& result) {
+    result = "TextEdit";
+}
+
 void TextEdit::set_cursor(uint32_t pos) {
     cursor_pos_ = std::min<size_t>(pos, text_.length());
     set_dirty();
@@ -1890,6 +1997,13 @@ const std::string& TextField::get_text() const {
     return text;
 }
 
+void TextField::getAccessibilityText(std::string& result) {
+    result = text;
+}
+void TextField::getWidgetName(std::string& result) {
+    result = "TextField";
+}
+
 void TextField::set_text(std::string_view value) {
     set(value);
     if (on_change)
@@ -1945,6 +2059,13 @@ int32_t NumberField::value() const {
     return value_;
 }
 
+void NumberField::getAccessibilityText(std::string& result) {
+    result = to_string_dec_int(value_);
+}
+void NumberField::getWidgetName(std::string& result) {
+    result = "NumberField";
+}
+
 void NumberField::set_value(int32_t new_value, bool trigger_change) {
     if (can_loop) {
         if (new_value >= range.first)
@@ -2155,6 +2276,12 @@ const std::string& SymField::to_string() const {
     return value_;
 }
 
+void SymField::getAccessibilityText(std::string& result) {
+    result = value_;
+}
+void SymField::getWidgetName(std::string& result) {
+    result = "SymField";
+}
 void SymField::paint(Painter& painter) {
     Point p = screen_pos();
 

firmware/common/ui_widget.hpp

@@ -106,6 +106,9 @@ class Widget {
 
     virtual Context& context() const;
 
+    virtual void getAccessibilityText(std::string& result);
+    virtual void getWidgetName(std::string& result);
+
     void set_style(const Style* new_style);
 
     const Style& style() const;
@@ -207,6 +210,8 @@ class Text : public Widget {
     void set(std::string_view value);
 
     void paint(Painter& painter) override;
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
 
    protected:
     // NB: Don't truncate this string. The UI will only render
@@ -234,6 +239,8 @@ class Labels : public Widget {
     void set_labels(std::initializer_list<Label> labels);
 
     void paint(Painter& painter) override;
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
 
    private:
     std::vector<Label> labels_;
@@ -312,6 +319,8 @@ class ProgressBar : public Widget {
     void set_value(const uint32_t value);
 
     void paint(Painter& painter) override;
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
 
    private:
     uint32_t _value = 0;
@@ -345,6 +354,8 @@ class Console : public Widget {
     void enable_scrolling(bool enable);
     void on_show() override;
     void on_hide() override;
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
 
    private:
     Point pos{0, 0};
@@ -385,6 +396,8 @@ class Checkbox : public Widget {
     bool on_key(const KeyEvent key) override;
     bool on_keyboard(const KeyboardEvent key) override;
     bool on_touch(const TouchEvent event) override;
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
 
    private:
     std::string text_;
@@ -421,6 +434,8 @@ class Button : public Widget {
     bool on_key(const KeyEvent key) override;
     bool on_touch(const TouchEvent event) override;
     bool on_keyboard(const KeyboardEvent event) override;
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
 
    private:
     std::string text_;
@@ -461,6 +476,9 @@ class ButtonWithEncoder : public Widget {
     bool on_encoder(const EncoderEvent delta) override;
     bool on_keyboard(const KeyboardEvent event) override;
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
    private:
     std::string text_;
     int32_t encoder_delta = 0;
@@ -496,6 +514,9 @@ class NewButton : public Widget {
     bool on_touch(const TouchEvent event) override;
     bool on_keyboard(const KeyboardEvent event) override;
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
     void paint(Painter& painter) override;
 
    protected:
@@ -550,6 +571,8 @@ class ImageButton : public Image {
     bool on_key(const KeyEvent key) override;
     bool on_touch(const TouchEvent event) override;
     bool on_keyboard(const KeyboardEvent event) override;
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
 };
 
 /* A button that toggles between two images when set. */
@@ -583,6 +606,9 @@ class ImageToggle : public ImageButton {
     bool value() const;
     void set_value(bool b);
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
    private:
     // Hide this field.
     using ImageButton::on_select;
@@ -628,6 +654,8 @@ class ImageOptionsField : public Widget {
     bool on_encoder(const EncoderEvent delta) override;
     bool on_touch(const TouchEvent event) override;
     bool on_keyboard(const KeyboardEvent event) override;
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
 
    private:
     options_t options;
@@ -667,6 +695,9 @@ class OptionsField : public Widget {
     bool on_touch(const TouchEvent event) override;
     bool on_keyboard(const KeyboardEvent event) override;
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
    private:
     const size_t length_;
     options_t options_;
@@ -712,6 +743,9 @@ class TextEdit : public Widget {
     bool on_touch(const TouchEvent event) override;
     bool on_keyboard(const KeyboardEvent event) override;
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
     void on_focus() override;
     void on_blur() override;
 
@@ -738,6 +772,9 @@ class TextField : public Text {
     bool on_encoder(EncoderEvent delta) override;
     bool on_touch(TouchEvent event) override;
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
    private:
     using Text::set;
 };
@@ -774,6 +811,9 @@ class NumberField : public Widget {
     bool on_touch(const TouchEvent event) override;
     bool on_keyboard(const KeyboardEvent event) override;
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
    private:
     range_t range;
     int32_t step;
@@ -849,6 +889,9 @@ class SymField : public Widget {
     bool on_encoder(EncoderEvent delta) override;
     bool on_touch(TouchEvent event) override;
 
+    void getAccessibilityText(std::string& result) override;
+    void getWidgetName(std::string& result) override;
+
    private:
     /* Ensure the specified symbol is in the symbol list. */
     char ensure_valid(char symbol) const;