Oversample capturing for low bandwidths (#1332)

* draft_low_bit_rate_solution_Capture_App

* second_draft_dynamic_decim

* Add support for Oversample Rate to capture.

---------

Co-authored-by: Brumi-2021 <ea3hqj@gmail.com>

firmware/application/apps/capture_app.cpp

@@ -64,13 +64,19 @@ CaptureAppView::CaptureAppView(NavigationView& nav)
         /* base_rate is used for FFT calculation and display LCD, and also in recording writing SD Card rate. */
         /* ex. sampling_rate values, 4Mhz, when recording 500 kHz (BW) and fs 8 Mhz, when selected 1 Mhz BW ... */
         /* ex. recording 500kHz BW to .C16 file, base_rate clock 500kHz x2(I,Q) x 2 bytes (int signed) =2MB/sec rate SD Card  */
-        auto sampling_rate = 8 * base_rate;  // Decimation by 8 done on baseband side.
 
-        /* Set up proper anti aliasing BPF bandwith in MAX2837 before ADC sampling according to the new added BW Options. */
+        // For lower bandwidths, (12k5, 16k, 20k), increase the oversample rate to get a higher sample rate.
+        OversampleRate oversample_rate = base_rate >= 25'000 ? OversampleRate::Rate8x : OversampleRate::Rate16x;
+
+        // HackRF suggests a minimum sample rate of 2M.
+        // Oversampling helps get to higher sample rates when recording lower bandwidths.
+        uint32_t sampling_rate = toUType(oversample_rate) * base_rate;
+
+        // Set up proper anti aliasing BPF bandwidth in MAX2837 before ADC sampling according to the new added BW Options.
         auto anti_alias_baseband_bandwidth_filter = filter_bandwidth_for_sampling_rate(sampling_rate);
 
         waterfall.stop();
-        record_view.set_sampling_rate(sampling_rate);
+        record_view.set_sampling_rate(sampling_rate, oversample_rate);  // NB: Actually updates the baseband.
         receiver_model.set_sampling_rate(sampling_rate);
         receiver_model.set_baseband_bandwidth(anti_alias_baseband_bandwidth_filter);
         waterfall.start();

firmware/application/apps/ui_recon.cpp

@@ -1251,8 +1251,9 @@ size_t ReconView::change_mode(freqman_index_t new_mod) {
     }
     if (new_mod != SPEC_MODULATION) {
         button_audio_app.set_text("AUDIO");
+        // TODO: Oversampling.
         record_view->set_sampling_rate(recording_sampling_rate);
-        // reset receiver model to fix bug when going from SPEC to audio, the sound is distorded
+        // reset receiver model to fix bug when going from SPEC to audio, the sound is distorted
         receiver_model.set_sampling_rate(3072000);
         receiver_model.set_baseband_bandwidth(1750000);
     } else {

firmware/application/baseband_api.cpp

@@ -352,6 +352,11 @@ void set_sample_rate(const uint32_t sample_rate) {
     send_message(&message);
 }
 
+void set_oversample_rate(OversampleRate oversample_rate) {
+    OversampleRateConfigMessage message{oversample_rate};
+    send_message(&message);
+}
+
 void capture_start(CaptureConfig* const config) {
     CaptureConfigMessage message{config};
     send_message(&message);

firmware/application/baseband_api.hpp

@@ -95,6 +95,7 @@ void spectrum_streaming_start();
 void spectrum_streaming_stop();
 
 void set_sample_rate(const uint32_t sample_rate);
+void set_oversample_rate(OversampleRate oversample_rate);
 void capture_start(CaptureConfig* const config);
 void capture_stop();
 void replay_start(ReplayConfig* const config);

firmware/application/freqman_db.cpp

@@ -74,9 +74,9 @@ options_t freqman_bandwidths[4] = {
     },
     {
         // SPEC -- TODO: these should be indexes.
-        {"8k5", 8500},
-        {"11k", 11000},
+        {"12k5", 12500},
         {"16k", 16000},
+        {"20k", 20000},
         {"25k", 25000},
         {"50k", 50000},
         {"100k", 100000},

firmware/application/ui_record_view.cpp

@@ -101,24 +101,27 @@ void RecordView::focus() {
     button_record.focus();
 }
 
-void RecordView::set_sampling_rate(const size_t new_sampling_rate) {
-    /* We are changing "REC" icon background to yellow in  BW rec Options >600kHz
+void RecordView::set_sampling_rate(size_t new_sampling_rate, OversampleRate new_oversample_rate) {
+    /* We are changing "REC" icon background to yellow in BW rec Options >600kHz
         where we are NOT recording full IQ .C16 files (recorded files are decimated ones).
-        Those decimated recorded files,has not the full IQ  samples .
-        are ok as recorded spectrum indication, but they  should not be used by Replay app.
+        Those decimated recorded files, has not the full IQ samples.
+        are ok as recorded spectrum indication, but they should not be used by Replay app.
 
-        We keep original black  background in all the correct IQ .C16 files BW's Options */
-    if (new_sampling_rate > 4800000) {  // > BW >600kHz  (fs=8*BW), (750kHz ...2750kHz)
+        We keep original black background in all the correct IQ .C16 files BW's Options */
+    if (new_sampling_rate > 4'800'000) {  // > BW >600kHz  (fs=8*BW), (750kHz...2750kHz)
         button_record.set_background(ui::Color::yellow());
     } else {
         button_record.set_background(ui::Color::black());
     }
 
-    if (new_sampling_rate != sampling_rate) {
+    if (new_sampling_rate != sampling_rate ||
+        new_oversample_rate != oversample_rate) {
         stop();
 
         sampling_rate = new_sampling_rate;
+        oversample_rate = new_oversample_rate;
         baseband::set_sample_rate(sampling_rate);
+        baseband::set_oversample_rate(oversample_rate);
 
         button_record.hidden(sampling_rate == 0);
         text_record_filename.hidden(sampling_rate == 0);
@@ -162,12 +165,13 @@ void RecordView::start() {
         rtcGetTime(&RTCD1, &datetime);
 
         // ISO 8601
-        std::string date_time = to_string_dec_uint(datetime.year(), 4, '0') +
-                                to_string_dec_uint(datetime.month(), 2, '0') +
-                                to_string_dec_uint(datetime.day(), 2, '0') + "T" +
-                                to_string_dec_uint(datetime.hour()) +
-                                to_string_dec_uint(datetime.minute()) +
-                                to_string_dec_uint(datetime.second());
+        std::string date_time =
+            to_string_dec_uint(datetime.year(), 4, '0') +
+            to_string_dec_uint(datetime.month(), 2, '0') +
+            to_string_dec_uint(datetime.day(), 2, '0') + "T" +
+            to_string_dec_uint(datetime.hour()) +
+            to_string_dec_uint(datetime.minute()) +
+            to_string_dec_uint(datetime.second());
 
         base_path = filename_stem_pattern.string() + "_" + date_time + "_" +
                     trim(to_string_freq(receiver_model.target_frequency())) + "Hz";
@@ -197,10 +201,9 @@ void RecordView::start() {
 
         case FileType::RawS8:
         case FileType::RawS16: {
-            const auto metadata_file_error =
-                write_metadata_file(get_metadata_path(base_path),
-                                    {receiver_model.target_frequency(), sampling_rate / 8});
-            // Not sure why sample_rate is div. 8, but stored value matches rate settings.
+            const auto metadata_file_error = write_metadata_file(
+                get_metadata_path(base_path),
+                {receiver_model.target_frequency(), sampling_rate / toUType(oversample_rate)});
             if (metadata_file_error.is_valid()) {
                 handle_error(metadata_file_error.value());
                 return;
@@ -263,18 +266,24 @@ void RecordView::update_status_display() {
         text_record_dropped.set(s);
     }
 
-    /*if (pitch_rssi_enabled) {
-                button_pitch_rssi.invert_colors();
-        }*/
+    /*
+    if (pitch_rssi_enabled) {
+        button_pitch_rssi.invert_colors();
+    }
+    */
 
-    if (sampling_rate) {
+    if (sampling_rate > 0) {
         const auto space_info = std::filesystem::space(u"");
-        const uint32_t bytes_per_second =
-            // - Audio is 1 int16_t per sample or '2' bytes per sample.
-            // - C8 captures 2 (I,Q) int8_t per sample or '2' bytes per sample.
-            // - C16 captures 2 (I,Q) int16_t per sample or '4' bytes per sample.
-            //   Dividing to get actual sample rate because of decimation in proc_capture.
-            file_type == FileType::WAV ? (sampling_rate * 2) : (sampling_rate * ((file_type == FileType::RawS8) ? 2 : 4) / 8);
+        // - Audio is 1 int16_t per sample or '2' bytes per sample.
+        // - C8 captures 2 (I,Q) int8_t per sample or '2' bytes per sample.
+        // - C16 captures 2 (I,Q) int16_t per sample or '4' bytes per sample.
+        const auto bytes_per_sample = file_type == FileType::RawS16 ? 4 : 2;
+        // WAV files are not oversampled, but C8 and C16 are. Divide by the
+        // oversample rate to get the effective sample rate.
+        const auto effective_sampling_rate = file_type == FileType::WAV
+                                                 ? sampling_rate
+                                                 : sampling_rate / toUType(oversample_rate);
+        const uint32_t bytes_per_second = effective_sampling_rate * bytes_per_sample;
         const uint32_t available_seconds = space_info.free / bytes_per_second;
         const uint32_t seconds = available_seconds % 60;
         const uint32_t available_minutes = available_seconds / 60;

firmware/application/ui_record_view.hpp

@@ -56,7 +56,16 @@ class RecordView : public View {
 
     void focus() override;
 
-    void set_sampling_rate(const size_t new_sampling_rate);
+    /* Sets the sampling rate and the oversampling "decimation" rate.
+     * These values are passed down to the baseband proc_capture. For
+     * Audio (WAV) recording, the OversampleRate should not be
+     * specified and the default will be used. */
+    /* TODO: Currently callers are expected to have already multiplied the
+     * sample_rate with the oversample rate. It would be better move that
+     * logic to a single place. */
+    void set_sampling_rate(
+        size_t new_sampling_rate,
+        OversampleRate new_oversample_rate = OversampleRate::Rate8x);
 
     void set_file_type(const FileType v) { file_type = v; }
 
@@ -90,6 +99,7 @@ class RecordView : public View {
     const size_t write_size;
     const size_t buffer_count;
     size_t sampling_rate{0};
+    OversampleRate oversample_rate{OversampleRate::Rate8x};
     SignalToken signal_token_tick_second{};
 
     Rectangle rect_background{

firmware/baseband/proc_capture.cpp

@@ -27,7 +27,8 @@
 #include "utility.hpp"
 
 CaptureProcessor::CaptureProcessor() {
-    decim_0.configure(taps_200k_decim_0.taps, 33554432);
+    decim_0_4.configure(taps_200k_decim_0.taps, 33554432);
+    decim_0_8.configure(taps_200k_decim_0.taps, 33554432);
     decim_1.configure(taps_200k_decim_1.taps, 131072);
 
     channel_spectrum.set_decimation_factor(1);
@@ -36,7 +37,7 @@ CaptureProcessor::CaptureProcessor() {
 
 void CaptureProcessor::execute(const buffer_c8_t& buffer) {
     /* 2.4576MHz, 2048 samples */
-    const auto decim_0_out = decim_0.execute(buffer, dst_buffer);
+    const auto decim_0_out = decim_0_execute(buffer, dst_buffer);
     const auto decim_1_out = decim_1.execute(decim_0_out, dst_buffer);
     const auto& decimator_out = decim_1_out;
     const auto& channel = decimator_out;
@@ -65,9 +66,19 @@ void CaptureProcessor::on_message(const Message* const message) {
             channel_spectrum.on_message(message);
             break;
 
-        case Message::ID::SamplerateConfig:
-            samplerate_config(*reinterpret_cast<const SamplerateConfigMessage*>(message));
+        case Message::ID::SamplerateConfig: {
+            auto config = reinterpret_cast<const SamplerateConfigMessage*>(message);
+            baseband_fs = config->sample_rate;
+            update_for_rate_change();
             break;
+        }
+
+        case Message::ID::OversampleRateConfig: {
+            auto config = reinterpret_cast<const OversampleRateConfigMessage*>(message);
+            oversample_rate = config->oversample_rate;
+            update_for_rate_change();
+            break;
+        }
 
         case Message::ID::CaptureConfig:
             capture_config(*reinterpret_cast<const CaptureConfigMessage*>(message));
@@ -78,11 +89,14 @@ void CaptureProcessor::on_message(const Message* const message) {
     }
 }
 
-void CaptureProcessor::samplerate_config(const SamplerateConfigMessage& message) {
-    baseband_fs = message.sample_rate;
+void CaptureProcessor::update_for_rate_change() {
     baseband_thread.set_sampling_rate(baseband_fs);
 
-    size_t decim_0_output_fs = baseband_fs / decim_0.decimation_factor;
+    auto decim_0_factor = oversample_rate == OversampleRate::Rate8x
+                              ? decim_0_4.decimation_factor
+                              : decim_0_8.decimation_factor;
+
+    size_t decim_0_output_fs = baseband_fs / decim_0_factor;
 
     size_t decim_1_input_fs = decim_0_output_fs;
     size_t decim_1_output_fs = decim_1_input_fs / decim_1.decimation_factor;
@@ -103,6 +117,20 @@ void CaptureProcessor::capture_config(const CaptureConfigMessage& message) {
     }
 }
 
+buffer_c16_t CaptureProcessor::decim_0_execute(const buffer_c8_t& src, const buffer_c16_t& dst) {
+    switch (oversample_rate) {
+        case OversampleRate::Rate8x:
+            return decim_0_4.execute(src, dst);
+
+        case OversampleRate::Rate16x:
+            return decim_0_8.execute(src, dst);
+
+        default:
+            chDbgPanic("Unhandled OversampleRate");
+            return {};
+    }
+}
+
 int main() {
     EventDispatcher event_dispatcher{std::make_unique<CaptureProcessor>()};
     event_dispatcher.run();

firmware/baseband/proc_capture.hpp

@@ -50,7 +50,13 @@ class CaptureProcessor : public BasebandProcessor {
         dst.data(),
         dst.size()};
 
-    dsp::decimate::FIRC8xR16x24FS4Decim4 decim_0{};
+    /* NB: There are two decimation passes: 0 and 1. In pass 0, one of
+     * the following will be selected based on the oversample rate.
+     * use decim_0_4 for an overall decimation factor of 8.
+     * use decim_0_8 for an overall decimation factor of 16. */
+    dsp::decimate::FIRC8xR16x24FS4Decim4 decim_0_4{};
+    dsp::decimate::FIRC8xR16x24FS4Decim8 decim_0_8{};
+
     dsp::decimate::FIRC16xR16x16Decim2 decim_1{};
     int32_t channel_filter_low_f = 0;
     int32_t channel_filter_high_f = 0;
@@ -61,14 +67,19 @@ class CaptureProcessor : public BasebandProcessor {
     SpectrumCollector channel_spectrum{};
     size_t spectrum_interval_samples = 0;
     size_t spectrum_samples = 0;
+    OversampleRate oversample_rate{OversampleRate::Rate8x};
 
     /* NB: Threads should be the last members in the class definition. */
     BasebandThread baseband_thread{
         baseband_fs, this, baseband::Direction::Receive, /*auto_start*/ false};
     RSSIThread rssi_thread{};
 
-    void samplerate_config(const SamplerateConfigMessage& message);
+    /* Called to update members when the sample rate or oversample rate is changed. */
+    void update_for_rate_change();
     void capture_config(const CaptureConfigMessage& message);
+
+    /* Dispatch to the correct decim_0 based on oversample rate. */
+    buffer_c16_t decim_0_execute(const buffer_c8_t& src, const buffer_c16_t& dst);
 };
 
 #endif /*__PROC_CAPTURE_HPP__*/

firmware/common/message.hpp

@@ -111,6 +111,7 @@ class Message {
         APRSRxConfigure = 54,
         SpectrumPainterBufferRequestConfigure = 55,
         SpectrumPainterBufferResponseConfigure = 56,
+        OversampleRateConfig = 57,
         MAX
     };
 
@@ -809,6 +810,23 @@ class SamplerateConfigMessage : public Message {
     const uint32_t sample_rate = 0;
 };
 
+/* Controls decimation handling in proc_capture. */
+enum class OversampleRate : uint8_t {
+    Rate8x = 8,
+    Rate16x = 16,
+};
+
+class OversampleRateConfigMessage : public Message {
+   public:
+    constexpr OversampleRateConfigMessage(
+        OversampleRate oversample_rate)
+        : Message{ID::OversampleRateConfig},
+          oversample_rate(oversample_rate) {
+    }
+
+    const OversampleRate oversample_rate{OversampleRate::Rate8x};
+};
+
 class AudioLevelReportMessage : public Message {
    public:
     constexpr AudioLevelReportMessage()