Oversample capturing for low bandwidths (#1332)

* draft_low_bit_rate_solution_Capture_App

* second_draft_dynamic_decim

* Add support for Oversample Rate to capture.

---------

Co-authored-by: Brumi-2021 <ea3hqj@gmail.com>

firmware/application/apps/capture_app.cpp

@@ -64,13 +64,19 @@ CaptureAppView::CaptureAppView(NavigationView& nav)
         /* base_rate is used for FFT calculation and display LCD, and also in recording writing SD Card rate. */
         /* ex. sampling_rate values, 4Mhz, when recording 500 kHz (BW) and fs 8 Mhz, when selected 1 Mhz BW ... */
         /* ex. recording 500kHz BW to .C16 file, base_rate clock 500kHz x2(I,Q) x 2 bytes (int signed) =2MB/sec rate SD Card  */
-        auto sampling_rate = 8 * base_rate;  // Decimation by 8 done on baseband side.
 
-        /* Set up proper anti aliasing BPF bandwith in MAX2837 before ADC sampling according to the new added BW Options. */
+        // For lower bandwidths, (12k5, 16k, 20k), increase the oversample rate to get a higher sample rate.
+        OversampleRate oversample_rate = base_rate >= 25'000 ? OversampleRate::Rate8x : OversampleRate::Rate16x;
+
+        // HackRF suggests a minimum sample rate of 2M.
+        // Oversampling helps get to higher sample rates when recording lower bandwidths.
+        uint32_t sampling_rate = toUType(oversample_rate) * base_rate;
+
+        // Set up proper anti aliasing BPF bandwidth in MAX2837 before ADC sampling according to the new added BW Options.
         auto anti_alias_baseband_bandwidth_filter = filter_bandwidth_for_sampling_rate(sampling_rate);
 
         waterfall.stop();
-        record_view.set_sampling_rate(sampling_rate);
+        record_view.set_sampling_rate(sampling_rate, oversample_rate);  // NB: Actually updates the baseband.
         receiver_model.set_sampling_rate(sampling_rate);
         receiver_model.set_baseband_bandwidth(anti_alias_baseband_bandwidth_filter);
         waterfall.start();

firmware/application/apps/ui_recon.cpp

@@ -1251,8 +1251,9 @@ size_t ReconView::change_mode(freqman_index_t new_mod) {
     }
     if (new_mod != SPEC_MODULATION) {
         button_audio_app.set_text("AUDIO");
+        // TODO: Oversampling.
         record_view->set_sampling_rate(recording_sampling_rate);
-        // reset receiver model to fix bug when going from SPEC to audio, the sound is distorded
+        // reset receiver model to fix bug when going from SPEC to audio, the sound is distorted
         receiver_model.set_sampling_rate(3072000);
         receiver_model.set_baseband_bandwidth(1750000);
     } else {

firmware/application/baseband_api.cpp

@@ -352,6 +352,11 @@ void set_sample_rate(const uint32_t sample_rate) {
     send_message(&message);
 }
 
+void set_oversample_rate(OversampleRate oversample_rate) {
+    OversampleRateConfigMessage message{oversample_rate};
+    send_message(&message);
+}
+
 void capture_start(CaptureConfig* const config) {
     CaptureConfigMessage message{config};
     send_message(&message);

firmware/application/baseband_api.hpp

@@ -95,6 +95,7 @@ void spectrum_streaming_start();
 void spectrum_streaming_stop();
 
 void set_sample_rate(const uint32_t sample_rate);
+void set_oversample_rate(OversampleRate oversample_rate);
 void capture_start(CaptureConfig* const config);
 void capture_stop();
 void replay_start(ReplayConfig* const config);

firmware/application/freqman_db.cpp

@@ -74,9 +74,9 @@ options_t freqman_bandwidths[4] = {
     },
     {
         // SPEC -- TODO: these should be indexes.
-        {"8k5", 8500},
+        {"12k5", 12500},
-        {"11k", 11000},
         {"16k", 16000},
+        {"20k", 20000},
         {"25k", 25000},
         {"50k", 50000},
         {"100k", 100000},

firmware/application/ui_record_view.cpp

@@ -101,24 +101,27 @@ void RecordView::focus() {
     button_record.focus();
 }
 
-void RecordView::set_sampling_rate(const size_t new_sampling_rate) {
+void RecordView::set_sampling_rate(size_t new_sampling_rate, OversampleRate new_oversample_rate) {
     /* We are changing "REC" icon background to yellow in BW rec Options >600kHz
         where we are NOT recording full IQ .C16 files (recorded files are decimated ones).
-        Those decimated recorded files,has not the full IQ  samples .
+        Those decimated recorded files, has not the full IQ samples.
         are ok as recorded spectrum indication, but they should not be used by Replay app.
 
         We keep original black background in all the correct IQ .C16 files BW's Options */
-    if (new_sampling_rate > 4800000) {  // > BW >600kHz  (fs=8*BW), (750kHz ...2750kHz)
+    if (new_sampling_rate > 4'800'000) {  // > BW >600kHz  (fs=8*BW), (750kHz...2750kHz)
         button_record.set_background(ui::Color::yellow());
     } else {
         button_record.set_background(ui::Color::black());
     }
 
-    if (new_sampling_rate != sampling_rate) {
+    if (new_sampling_rate != sampling_rate ||
+        new_oversample_rate != oversample_rate) {
         stop();
 
         sampling_rate = new_sampling_rate;
+        oversample_rate = new_oversample_rate;
         baseband::set_sample_rate(sampling_rate);
+        baseband::set_oversample_rate(oversample_rate);
 
         button_record.hidden(sampling_rate == 0);
         text_record_filename.hidden(sampling_rate == 0);
@@ -162,7 +165,8 @@ void RecordView::start() {
         rtcGetTime(&RTCD1, &datetime);
 
         // ISO 8601
-        std::string date_time = to_string_dec_uint(datetime.year(), 4, '0') +
+        std::string date_time =
+            to_string_dec_uint(datetime.year(), 4, '0') +
             to_string_dec_uint(datetime.month(), 2, '0') +
             to_string_dec_uint(datetime.day(), 2, '0') + "T" +
             to_string_dec_uint(datetime.hour()) +
@@ -197,10 +201,9 @@ void RecordView::start() {
 
         case FileType::RawS8:
         case FileType::RawS16: {
-            const auto metadata_file_error =
+            const auto metadata_file_error = write_metadata_file(
-                write_metadata_file(get_metadata_path(base_path),
+                get_metadata_path(base_path),
-                                    {receiver_model.target_frequency(), sampling_rate / 8});
+                {receiver_model.target_frequency(), sampling_rate / toUType(oversample_rate)});
-            // Not sure why sample_rate is div. 8, but stored value matches rate settings.
             if (metadata_file_error.is_valid()) {
                 handle_error(metadata_file_error.value());
                 return;
@@ -263,18 +266,24 @@ void RecordView::update_status_display() {
         text_record_dropped.set(s);
     }
 
-    /*if (pitch_rssi_enabled) {
+    /*
+    if (pitch_rssi_enabled) {
         button_pitch_rssi.invert_colors();
-        }*/
+    }
+    */
 
-    if (sampling_rate) {
+    if (sampling_rate > 0) {
         const auto space_info = std::filesystem::space(u"");
-        const uint32_t bytes_per_second =
         // - Audio is 1 int16_t per sample or '2' bytes per sample.
         // - C8 captures 2 (I,Q) int8_t per sample or '2' bytes per sample.
         // - C16 captures 2 (I,Q) int16_t per sample or '4' bytes per sample.
-            //   Dividing to get actual sample rate because of decimation in proc_capture.
+        const auto bytes_per_sample = file_type == FileType::RawS16 ? 4 : 2;
-            file_type == FileType::WAV ? (sampling_rate * 2) : (sampling_rate * ((file_type == FileType::RawS8) ? 2 : 4) / 8);
+        // WAV files are not oversampled, but C8 and C16 are. Divide by the
+        // oversample rate to get the effective sample rate.
+        const auto effective_sampling_rate = file_type == FileType::WAV
+                                                 ? sampling_rate
+                                                 : sampling_rate / toUType(oversample_rate);
+        const uint32_t bytes_per_second = effective_sampling_rate * bytes_per_sample;
         const uint32_t available_seconds = space_info.free / bytes_per_second;
         const uint32_t seconds = available_seconds % 60;
         const uint32_t available_minutes = available_seconds / 60;

firmware/application/ui_record_view.hpp

@@ -56,7 +56,16 @@ class RecordView : public View {
 
     void focus() override;
 
-    void set_sampling_rate(const size_t new_sampling_rate);
+    /* Sets the sampling rate and the oversampling "decimation" rate.
+     * These values are passed down to the baseband proc_capture. For
+     * Audio (WAV) recording, the OversampleRate should not be
+     * specified and the default will be used. */
+    /* TODO: Currently callers are expected to have already multiplied the
+     * sample_rate with the oversample rate. It would be better move that
+     * logic to a single place. */
+    void set_sampling_rate(
+        size_t new_sampling_rate,
+        OversampleRate new_oversample_rate = OversampleRate::Rate8x);
 
     void set_file_type(const FileType v) { file_type = v; }
 
@@ -90,6 +99,7 @@ class RecordView : public View {
     const size_t write_size;
     const size_t buffer_count;
     size_t sampling_rate{0};
+    OversampleRate oversample_rate{OversampleRate::Rate8x};
     SignalToken signal_token_tick_second{};
 
     Rectangle rect_background{

firmware/baseband/proc_capture.cpp

@@ -27,7 +27,8 @@
 #include "utility.hpp"
 
 CaptureProcessor::CaptureProcessor() {
-    decim_0.configure(taps_200k_decim_0.taps, 33554432);
+    decim_0_4.configure(taps_200k_decim_0.taps, 33554432);
+    decim_0_8.configure(taps_200k_decim_0.taps, 33554432);
     decim_1.configure(taps_200k_decim_1.taps, 131072);
 
     channel_spectrum.set_decimation_factor(1);
@@ -36,7 +37,7 @@ CaptureProcessor::CaptureProcessor() {
 
 void CaptureProcessor::execute(const buffer_c8_t& buffer) {
     /* 2.4576MHz, 2048 samples */
-    const auto decim_0_out = decim_0.execute(buffer, dst_buffer);
+    const auto decim_0_out = decim_0_execute(buffer, dst_buffer);
     const auto decim_1_out = decim_1.execute(decim_0_out, dst_buffer);
     const auto& decimator_out = decim_1_out;
     const auto& channel = decimator_out;
@@ -65,9 +66,19 @@ void CaptureProcessor::on_message(const Message* const message) {
             channel_spectrum.on_message(message);
             break;
 
-        case Message::ID::SamplerateConfig:
+        case Message::ID::SamplerateConfig: {
-            samplerate_config(*reinterpret_cast<const SamplerateConfigMessage*>(message));
+            auto config = reinterpret_cast<const SamplerateConfigMessage*>(message);
+            baseband_fs = config->sample_rate;
+            update_for_rate_change();
             break;
+        }
+
+        case Message::ID::OversampleRateConfig: {
+            auto config = reinterpret_cast<const OversampleRateConfigMessage*>(message);
+            oversample_rate = config->oversample_rate;
+            update_for_rate_change();
+            break;
+        }
 
         case Message::ID::CaptureConfig:
             capture_config(*reinterpret_cast<const CaptureConfigMessage*>(message));
@@ -78,11 +89,14 @@ void CaptureProcessor::on_message(const Message* const message) {
     }
 }
 
-void CaptureProcessor::samplerate_config(const SamplerateConfigMessage& message) {
+void CaptureProcessor::update_for_rate_change() {
-    baseband_fs = message.sample_rate;
     baseband_thread.set_sampling_rate(baseband_fs);
 
-    size_t decim_0_output_fs = baseband_fs / decim_0.decimation_factor;
+    auto decim_0_factor = oversample_rate == OversampleRate::Rate8x
+                              ? decim_0_4.decimation_factor
+                              : decim_0_8.decimation_factor;
+
+    size_t decim_0_output_fs = baseband_fs / decim_0_factor;
 
     size_t decim_1_input_fs = decim_0_output_fs;
     size_t decim_1_output_fs = decim_1_input_fs / decim_1.decimation_factor;
@@ -103,6 +117,20 @@ void CaptureProcessor::capture_config(const CaptureConfigMessage& message) {
     }
 }
 
+buffer_c16_t CaptureProcessor::decim_0_execute(const buffer_c8_t& src, const buffer_c16_t& dst) {
+    switch (oversample_rate) {
+        case OversampleRate::Rate8x:
+            return decim_0_4.execute(src, dst);
+
+        case OversampleRate::Rate16x:
+            return decim_0_8.execute(src, dst);
+
+        default:
+            chDbgPanic("Unhandled OversampleRate");
+            return {};
+    }
+}
+
 int main() {
     EventDispatcher event_dispatcher{std::make_unique<CaptureProcessor>()};
     event_dispatcher.run();

firmware/baseband/proc_capture.hpp

@@ -50,7 +50,13 @@ class CaptureProcessor : public BasebandProcessor {
         dst.data(),
         dst.size()};
 
-    dsp::decimate::FIRC8xR16x24FS4Decim4 decim_0{};
+    /* NB: There are two decimation passes: 0 and 1. In pass 0, one of
+     * the following will be selected based on the oversample rate.
+     * use decim_0_4 for an overall decimation factor of 8.
+     * use decim_0_8 for an overall decimation factor of 16. */
+    dsp::decimate::FIRC8xR16x24FS4Decim4 decim_0_4{};
+    dsp::decimate::FIRC8xR16x24FS4Decim8 decim_0_8{};
+
     dsp::decimate::FIRC16xR16x16Decim2 decim_1{};
     int32_t channel_filter_low_f = 0;
     int32_t channel_filter_high_f = 0;
@@ -61,14 +67,19 @@ class CaptureProcessor : public BasebandProcessor {
     SpectrumCollector channel_spectrum{};
     size_t spectrum_interval_samples = 0;
     size_t spectrum_samples = 0;
+    OversampleRate oversample_rate{OversampleRate::Rate8x};
 
     /* NB: Threads should be the last members in the class definition. */
     BasebandThread baseband_thread{
         baseband_fs, this, baseband::Direction::Receive, /*auto_start*/ false};
     RSSIThread rssi_thread{};
 
-    void samplerate_config(const SamplerateConfigMessage& message);
+    /* Called to update members when the sample rate or oversample rate is changed. */
+    void update_for_rate_change();
     void capture_config(const CaptureConfigMessage& message);
+
+    /* Dispatch to the correct decim_0 based on oversample rate. */
+    buffer_c16_t decim_0_execute(const buffer_c8_t& src, const buffer_c16_t& dst);
 };
 
 #endif /*__PROC_CAPTURE_HPP__*/

firmware/common/message.hpp

@@ -111,6 +111,7 @@ class Message {
         APRSRxConfigure = 54,
         SpectrumPainterBufferRequestConfigure = 55,
         SpectrumPainterBufferResponseConfigure = 56,
+        OversampleRateConfig = 57,
         MAX
     };
 
@@ -809,6 +810,23 @@ class SamplerateConfigMessage : public Message {
     const uint32_t sample_rate = 0;
 };
 
+/* Controls decimation handling in proc_capture. */
+enum class OversampleRate : uint8_t {
+    Rate8x = 8,
+    Rate16x = 16,
+};
+
+class OversampleRateConfigMessage : public Message {
+   public:
+    constexpr OversampleRateConfigMessage(
+        OversampleRate oversample_rate)
+        : Message{ID::OversampleRateConfig},
+          oversample_rate(oversample_rate) {
+    }
+
+    const OversampleRate oversample_rate{OversampleRate::Rate8x};
+};
+
 class AudioLevelReportMessage : public Message {
    public:
     constexpr AudioLevelReportMessage()