Git-Mirrors/portapack-mayhem
1
0
Fork 0
mirror of https://github.com/eried/portapack-mayhem.git synced 2024-09-19 23:55:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved trimming (#1458)

Browse Source 
* Add threshold UI
* WIP Better trimming
* Rewrite mostly done WIP
* WIP - trim idea
* WIP threshold trimming
* WIP with new design
* Cleanup
This commit is contained in:
Kyle Reed 2023-09-23 12:56:37 -07:00 committed by GitHub
parent ef03f020ce
commit a6a1483083
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 417 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
firmware/application/CMakeLists.txt
View File

@ -182,6 +182,7 @@ set(CPPSRC
io_convert.cpp
io_file.cpp
io_wave.cpp
iq_trim.cpp
irq_controls.cpp
irq_lcd_frame.cpp
irq_rtc.cpp

134
firmware/application/apps/ui_iq_trim.cpp
View File

@ -30,41 +30,72 @@ namespace fs = std::filesystem;
namespace ui {
IQTrimView::IQTrimView(NavigationView& nav) {
IQTrimView::IQTrimView(NavigationView& nav)
: nav_{nav} {
add_children({
&labels,
&field_path,
&text_range,
&field_start,
&field_end,
&text_samples,
&text_max,
&field_cutoff,
&button_trim,
});
field_path.on_select = [this, &nav](TextField&) {
auto open_view = nav.push<FileLoadView>(".C*");
field_path.on_select = [this](TextField&) {
auto open_view = nav_.push<FileLoadView>(".C*");
open_view->push_dir(u"CAPTURES");
open_view->on_changed = [this](fs::path path) {
read_capture(path);
path_ = std::move(path);
profile_capture();
compute_range();
refresh_ui();
};
};
button_trim.on_select = [this, &nav](Button&) {
if (!path_.empty() && trim_range_.file_size > 0) {
progress_ui.show_trimming();
TrimFile(path_, trim_range_);
read_capture(path_);
text_samples.set_style(&Styles::light_grey);
text_max.set_style(&Styles::light_grey);
field_start.on_change = [this](int32_t v) {
if (field_end.value() < v)
field_end.set_value(v, false);
set_dirty();
};
field_end.on_change = [this](int32_t v) {
if (field_start.value() > v)
field_start.set_value(v, false);
set_dirty();
};
field_cutoff.set_value(7); // 7% of max is a good default.
field_cutoff.on_change = [this](int32_t) {
compute_range();
refresh_ui();
};
button_trim.on_select = [this](Button&) {
if (trim_capture()) {
profile_capture();
compute_range();
refresh_ui();
} else {
nav.display_modal("Error", "Select a file first.");
}
};
}
void IQTrimView::paint(Painter& painter) {
if (!path_.empty()) {
if (info_) {
uint32_t power_cutoff = field_cutoff.value() * static_cast<uint64_t>(info_->max_power) / 100;
// Draw power buckets.
for (size_t i = 0; i < power_buckets_.size(); ++i) {
auto amp = power_buckets_[i].power;
auto power = power_buckets_[i].power;
uint8_t amp = 0;
if (power > power_cutoff && info_->max_power > 0)
amp = (255ULL * power) / info_->max_power;
painter.draw_vline(
pos_lines + Point{(int)i, 0},
height_lines,
@ -72,8 +103,8 @@ void IQTrimView::paint(Painter& painter) {
}
// Draw trim range edges.
int start_x = screen_width * trim_range_.start / trim_range_.file_size;
int end_x = screen_width * trim_range_.end / trim_range_.file_size;
int start_x = screen_width * field_start.value() / info_->sample_count;
int end_x = screen_width * field_end.value() / info_->sample_count;
painter.draw_vline(
pos_lines + Point{start_x, 0},
@ -92,27 +123,72 @@ void IQTrimView::focus() {
void IQTrimView::refresh_ui() {
field_path.set_text(path_.filename().string());
text_range.set(to_string_dec_uint(trim_range_.start) + "-" + to_string_dec_uint(trim_range_.end));
text_samples.set(to_string_dec_uint(info_->sample_count));
text_max.set(to_string_dec_uint(info_->max_power));
set_dirty();
}
bool IQTrimView::read_capture(const fs::path& path) {
void IQTrimView::update_range_controls(iq::TrimRange trim_range) {
auto max_range = info_ ? info_->sample_count : 0;
auto step = info_ ? info_->sample_count / screen_width : 0;
field_start.set_range(0, max_range);
field_start.set_step(step);
field_end.set_range(0, max_range);
field_end.set_step(step);
field_start.set_value(trim_range.start_sample, false);
field_end.set_value(trim_range.end_sample, false);
}
void IQTrimView::profile_capture() {
power_buckets_ = {};
PowerBuckets buckets{
iq::PowerBuckets buckets{
.p = power_buckets_.data(),
.size = power_buckets_.size()};
progress_ui.show_reading();
auto range = ComputeTrimRange(path, amp_threshold, &buckets, progress_ui.get_callback());
info_ = iq::profile_capture(path_, buckets);
progress_ui.clear();
if (range) {
trim_range_ = *range;
return true;
} else {
trim_range_ = {};
return false;
}
}
} // namespace ui
void IQTrimView::compute_range() {
if (!info_)
return;
iq::PowerBuckets buckets{
.p = power_buckets_.data(),
.size = power_buckets_.size()};
auto trim_range = iq::compute_trim_range(*info_, buckets, field_cutoff.value());
update_range_controls(trim_range);
}
bool IQTrimView::trim_capture() {
if (!info_) {
nav_.display_modal("Error", "Open a file first.");
return false;
}
bool trimmed = false;
iq::TrimRange trim_range{
static_cast<uint32_t>(field_start.value()),
static_cast<uint32_t>(field_end.value()),
info_->sample_size};
if (trim_range.start_sample >= trim_range.end_sample) {
nav_.display_modal("Error", "Invalid trimming range.");
return false;
}
progress_ui.show_trimming();
trimmed = iq::trim_capture_with_range(path_, trim_range, progress_ui.get_callback());
progress_ui.clear();
if (!trimmed)
nav_.display_modal("Error", "Trimming failed.");
return trimmed;
}
} // namespace ui

64
firmware/application/apps/ui_iq_trim.hpp
View File

@ -24,6 +24,7 @@
#include "file.hpp"
#include "iq_trim.hpp"
#include "optional.hpp"
#include "ui.hpp"
#include "ui_navigation.hpp"
#include "ui_styles.hpp"
@ -49,7 +50,7 @@ class TrimProgressUI {
void show_progress(uint8_t percent) {
auto width = percent * screen_width / 100;
p.draw_hline({0, 6 * 16}, width, Color::yellow());
p.draw_hline({0, 6 * 16 + 2}, width, Color::yellow());
}
void clear() {
@ -73,18 +74,36 @@ class IQTrimView : public View {
void focus() override;
private:
/* Update controls with latest values. */
void refresh_ui();
bool read_capture(const std::filesystem::path& path);
/* Update the start/end controls with trim range info. */
void update_range_controls(iq::TrimRange trim_range);
/* Collect capture info and samples to draw the UI. */
void profile_capture();
/* Determine the start and end buckets based on the cutoff. */
void compute_range();
/* Trims the capture file based on the settings. */
bool trim_capture();
NavigationView& nav_;
std::filesystem::path path_{};
TrimRange trim_range_{};
std::array<PowerBuckets::Bucket, screen_width> power_buckets_{};
uint8_t amp_threshold = 5;
Optional<iq::CaptureInfo> info_{};
std::array<iq::PowerBuckets::Bucket, screen_width> power_buckets_{};
TrimProgressUI progress_ui{};
Labels labels{
{{0 * 8, 0 * 16}, "Capture File:", Color::light_grey()},
{{0 * 8, 6 * 16}, "Range:", Color::light_grey()},
{{0 * 8, 6 * 16}, "Start :", Color::light_grey()},
{{0 * 8, 7 * 16}, "End :", Color::light_grey()},
{{0 * 8, 8 * 16}, "Samples:", Color::light_grey()},
{{0 * 8, 9 * 16}, "Max Pwr:", Color::light_grey()},
{{0 * 8, 10 * 16}, "Cutoff :", Color::light_grey()},
{{12 * 8, 10 * 16}, "%", Color::light_grey()},
};
TextField field_path{
@ -94,12 +113,37 @@ class IQTrimView : public View {
Point pos_lines{0 * 8, 4 * 16};
Dim height_lines{2 * 16};
Text text_range{
{7 * 8, 6 * 16, 20 * 8, 1 * 16},
{}};
NumberField field_start{
{9 * 8, 6 * 16},
10,
{0, 0},
1,
' '};
NumberField field_end{
{9 * 8, 7 * 16},
10,
{0, 0},
1,
' '};
Text text_samples{
{9 * 8, 8 * 16, 10 * 8, 1 * 16},
"0"};
Text text_max{
{9 * 8, 9 * 16, 10 * 8, 1 * 16},
"0"};
NumberField field_cutoff{
{9 * 8, 10 * 16},
3,
{1, 100},
1,
' '};
Button button_trim{
{11 * 8, 9 * 16, 8 * 8, 3 * 16},
{20 * 8, 16 * 16, 8 * 8, 2 * 16},
"Trim"};
};

40
firmware/application/file.cpp
View File

@ -279,46 +279,6 @@ std::vector<std::filesystem::path> scan_root_directories(const std::filesystem::
return directory_list;
}
std::filesystem::filesystem_error trim_file(const std::filesystem::path& file_path, uint64_t start, uint64_t length) {
constexpr size_t buffer_size = std::filesystem::max_file_block_size;
uint8_t buffer[buffer_size];
auto temp_path = file_path + u"-tmp";
/* Scope for File instances. */
{
File src;
File dst;
auto error = src.open(file_path);
if (error) return error.value();
error = dst.create(temp_path);
if (error) return error.value();
src.seek(start);
auto remaining = length;
while (true) {
auto result = src.read(buffer, buffer_size);
if (result.is_error()) return result.error();
auto to_write = std::min(remaining, *result);
result = dst.write(buffer, to_write);
if (result.is_error()) return result.error();
remaining -= *result;
if (*result < buffer_size || remaining == 0)
break;
}
}
// Delete original and overwrite with temp file.
delete_file(file_path);
return rename_file(temp_path, file_path);
}
std::filesystem::filesystem_error delete_file(const std::filesystem::path& file_path) {
return {f_unlink(reinterpret_cast<const TCHAR*>(file_path.c_str()))};
}

2
firmware/application/file.hpp
View File

@ -265,7 +265,6 @@ struct FATTimestamp {
uint16_t FAT_time;
};
std::filesystem::filesystem_error trim_file(const std::filesystem::path& file_path, uint64_t start, uint64_t length);
std::filesystem::filesystem_error delete_file(const std::filesystem::path& file_path);
std::filesystem::filesystem_error rename_file(const std::filesystem::path& file_path, const std::filesystem::path& new_name);
std::filesystem::filesystem_error copy_file(const std::filesystem::path& file_path, const std::filesystem::path& dest_path);
@ -302,6 +301,7 @@ static_assert(sizeof(FIL::err) == 1, "FatFs FIL::err size not expected.");
#define FR_BAD_SEEK (0x102)
#define FR_UNEXPECTED (0x103)
/* NOTE: sizeof(File) == 556 bytes because of the FIL's buf member. */
class File {
public:
using Size = uint64_t;

195
firmware/application/iq_trim.cpp Normal file
View File

@ -0,0 +1,195 @@
/*
* Copyright (C) 2023 Kyle Reed
*
* This file is part of PortaPack.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; see the file COPYING. If not, write to
* the Free Software Foundation, Inc., 51 Franklin Street,
* Boston, MA 02110-1301, USA.
*/
#include "iq_trim.hpp"
#include <memory>
#include "string_format.hpp"
namespace fs = std::filesystem;
namespace iq {
/* Trimming helpers based on the sample type (complex8 or complex16). */
template <typename T>
uint32_t power(T value) {
auto real = value.real();
auto imag = value.imag();
return (real * real) + (imag * imag);
}
/* Collects capture file metadata and sample power buckets. */
template <typename T>
Optional<CaptureInfo> profile_capture(
const std::filesystem::path& path,
PowerBuckets& buckets,
uint8_t samples_per_bucket) {
File f;
auto error = f.open(path);
if (error)
return {};
CaptureInfo info{
.file_size = f.size(),
.sample_count = f.size() / sizeof(T),
.sample_size = sizeof(T),
.max_power = 0};
auto profile_samples = buckets.size * samples_per_bucket;
auto sample_interval = info.sample_count / profile_samples;
uint32_t bucket_width = std::max(1ULL, info.sample_count / buckets.size);
uint64_t sample_index = 0;
T value{};
while (true) {
f.seek(sample_index * info.sample_size);
auto result = f.read(&value, info.sample_size);
if (!result) return {}; // Read failed.
if (*result != info.sample_size)
break; // EOF
auto mag_squared = power(value);
if (mag_squared > info.max_power)
info.max_power = mag_squared;
auto bucket_index = sample_index / bucket_width;
buckets.add(bucket_index, mag_squared);
sample_index += sample_interval;
}
return info;
}
Optional<CaptureInfo> profile_capture(
const fs::path& path,
PowerBuckets& buckets,
uint8_t samples_per_bucket) {
auto sample_size = fs::capture_file_sample_size(path);
switch (sample_size) {
case sizeof(complex16_t):
return profile_capture<complex16_t>(path, buckets, samples_per_bucket);
case sizeof(complex8_t):
return profile_capture<complex8_t>(path, buckets, samples_per_bucket);
default:
return {};
};
}
TrimRange compute_trim_range(
CaptureInfo info,
const PowerBuckets& buckets,
uint8_t cutoff_percent) {
bool has_start = false;
uint8_t start_bucket = 0;
uint8_t end_bucket = 0;
uint32_t power_cutoff = cutoff_percent * static_cast<uint64_t>(info.max_power) / 100;
for (size_t i = 0; i < buckets.size; ++i) {
auto power = buckets.p[i].power;
if (power > power_cutoff) {
if (has_start)
end_bucket = i;
else {
start_bucket = i;
end_bucket = i;
has_start = true;
}
}
}
// End should be the first bucket after the last with signal.
// This makes the math downstream simpler. NB: may be > buckets.size.
end_bucket++;
auto samples_per_bucket = info.sample_count / buckets.size;
return {
start_bucket * samples_per_bucket,
end_bucket * samples_per_bucket,
info.sample_size};
}
bool trim_capture_with_range(
const fs::path& path,
TrimRange range,
const std::function<void(uint8_t)>& on_progress) {
constexpr size_t buffer_size = std::filesystem::max_file_block_size;
uint8_t buffer[buffer_size];
auto temp_path = path + u"-tmp";
// end_sample is the first sample to _not_ include.
auto start_byte = range.start_sample * range.sample_size;
auto end_byte = (range.end_sample * range.sample_size);
auto length = end_byte - start_byte;
// 'File' is 556 bytes! Heap alloc to avoid overflowing the stack.
auto src = std::make_unique<File>();
auto dst = std::make_unique<File>();
auto error = src->open(path);
if (error) return false;
error = dst->create(temp_path);
if (error) return false;
src->seek(start_byte);
auto processed = 0UL;
auto next_report = 0UL;
auto report_interval = length / 20UL;
while (true) {
auto result = src->read(buffer, buffer_size);
if (result.is_error()) return false;
auto remaining = length - processed;
auto to_write = std::min(remaining, *result);
result = dst->write(buffer, to_write);
if (result.is_error()) return false;
processed += *result;
if (*result < buffer_size || processed >= length)
break;
if (processed >= next_report) {
on_progress(100 * processed / length);
next_report += report_interval;
}
}
// Close files before renaming/deleting.
src.reset();
dst.reset();
// Delete original and overwrite with temp file.
delete_file(path);
rename_file(temp_path, path);
return true;
}
} // namespace iq

161
firmware/application/iq_trim.hpp
View File

@ -29,146 +29,65 @@
#include <functional>
#include <limits>
struct TrimRange {
uint64_t start;
uint64_t end;
uint64_t sample_count;
namespace iq {
/* Information about a capture. */
struct CaptureInfo {
uint64_t file_size;
uint64_t sample_count;
uint8_t sample_size;
uint32_t max_power;
};
/* Holds sample average power by bucket. */
struct PowerBuckets {
struct Bucket {
uint8_t power;
uint32_t power = 0;
uint8_t count = 0;
};
Bucket* p = nullptr;
size_t size = 0;
const size_t size = 0;
void add(size_t index) {
// This originally was meant to be an average power for the bucket,
// but it was a lot of extra math just for a little bit of UI and
// the math really slowed down processing. Instead, just count the
// number of samples above the threshold.
/* Add the power to the bucket average at index. */
void add(size_t index, uint32_t power) {
if (index < size) {
if (p[index].power < 255)
p[index].power++;
auto& b = p[index];
auto avg = static_cast<uint64_t>(b.power) * b.count;
b.count++;
b.power = (power + avg) / b.count;
}
}
};
inline bool TrimFile(const std::filesystem::path& path, TrimRange range) {
// NB: range.end should be included in the trimmed result, so '+ sample_size'.
auto result = trim_file(path, range.start, (range.end - range.start) + range.sample_size);
return result.ok();
}
template <typename T>
class IQTrimmer {
static constexpr uint8_t max_amp = 0xFF;
static constexpr typename T::value_type max_value = std::numeric_limits<typename T::value_type>::max();
static constexpr uint32_t max_mag_squared{2 * (max_value * max_value)}; // NB: Braces to detect narrowing.
public:
static Optional<TrimRange> ComputeTrimRange(
const std::filesystem::path& path,
uint8_t amp_threshold, // 0 - 255
PowerBuckets* buckets,
std::function<void(uint8_t)> on_progress) {
TrimRange range{};
File f;
auto error = f.open(path);
if (error)
return {};
constexpr size_t buffer_size = std::filesystem::max_file_block_size;
uint8_t buffer[buffer_size];
bool has_start = false;
size_t sample_index = 0;
File::Offset offset = 0;
uint8_t last_progress = 0;
size_t samples_per_bucket = 1;
// Scale from 0-255 to 0-max_mag_squared.
uint32_t threshold = (max_mag_squared * amp_threshold) / max_amp;
T value{};
range.file_size = f.size();
range.sample_size = sizeof(T);
range.sample_count = range.file_size / range.sample_size;
if (buckets)
samples_per_bucket = std::max(1ULL, range.sample_count / buckets->size);
while (true) {
auto result = f.read(buffer, buffer_size);
if (!result)
return {};
for (size_t i = 0; i < *result; i += sizeof(T)) {
++sample_index;
value = *reinterpret_cast<T*>(&buffer[i]);
auto real = value.real();
auto imag = value.imag();
uint32_t mag_squared = (real * real) + (imag * imag);
// Update range if above threshold.
if (mag_squared >= threshold) {
if (has_start) {
range.end = offset + i;
} else {
range.start = offset + i;
range.end = range.start;
has_start = true;
}
// Update the optional power bucket.
if (buckets) {
auto bucket_index = sample_index / samples_per_bucket;
buckets->add(bucket_index);
}
}
}
if (*result < buffer_size)
break;
offset += *result;
if (on_progress) {
uint8_t current_progress = 100 * offset / range.file_size;
if (last_progress != current_progress) {
on_progress(current_progress);
last_progress = current_progress;
}
}
}
return range;
}
/* Data needed to trim a capture by sample range.
* end_sample is the sample *after* the last to keep. */
struct TrimRange {
uint64_t start_sample;
uint64_t end_sample;
uint8_t sample_size;
};
inline Optional<TrimRange> ComputeTrimRange(
/* Collects capture file metadata and samples power buckets. */
Optional<CaptureInfo> profile_capture(
const std::filesystem::path& path,
uint8_t amp_threshold = 5,
PowerBuckets* buckets = nullptr,
std::function<void(uint8_t)> on_progress = nullptr) {
Optional<TrimRange> range;
auto sample_size = std::filesystem::capture_file_sample_size(path);
PowerBuckets& buckets,
uint8_t samples_per_bucket = 10);
switch (sample_size) {
case sizeof(complex16_t):
return IQTrimmer<complex16_t>::ComputeTrimRange(path, amp_threshold, buckets, on_progress);
/* Computes the trimming range given profiling info.
* Cutoff percent is a number 1-100. */
TrimRange compute_trim_range(
CaptureInfo info,
const PowerBuckets& buckets,
uint8_t cutoff_percent);
case sizeof(complex8_t):
return IQTrimmer<complex8_t>::ComputeTrimRange(path, amp_threshold, buckets, on_progress);
/* Trims the capture file with the specified range. */
bool trim_capture_with_range(
const std::filesystem::path& path,
TrimRange range,
const std::function<void(uint8_t)>& on_progress);
default:
return {};
};
}
} // namespace iq
#endif /*__IQ_TRIM_H__*/
#endif /*__IQ_TRIM_H__*/

27
firmware/application/ui_record_view.cpp
View File

@ -36,6 +36,7 @@ using namespace portapack;
#include "utility.hpp"
#include <cstdint>
#include <vector>
namespace ui {
@ -303,17 +304,25 @@ void RecordView::update_status_display() {
}
void RecordView::trim_capture() {
if (file_type != FileType::WAV && auto_trim && !trim_path.empty()) {
trim_ui.show_reading();
auto range = ComputeTrimRange(
trim_path,
/*threshold*/ 5,
/*power array*/ nullptr,
trim_ui.get_callback());
using bucket_t = iq::PowerBuckets::Bucket;
if (file_type != FileType::WAV && auto_trim && !trim_path.empty()) {
// Need to heap alloc the buckets in this case. The large static buffer overflows the stack.
std::vector<bucket_t> buckets(size_t(255), bucket_t{});
;
iq::PowerBuckets power_buckets{
.p = &buckets[0],
.size = buckets.size()};
trim_ui.show_reading();
auto info = iq::profile_capture(trim_path, power_buckets);
if (info) {
// 7% - decent trimming without being too aggressive.
auto trim_range = iq::compute_trim_range(*info, power_buckets, 7);
if (range) {
trim_ui.show_trimming();
TrimFile(trim_path, *range);
iq::trim_capture_with_range(trim_path, trim_range, trim_ui.get_callback());
}
trim_ui.clear();

3
firmware/common/ui_widget.cpp
View File

@ -1987,6 +1987,9 @@ void SymField::set_value(uint64_t value) {
value_[i] = uint_to_char(temp, radix);
v /= radix;
}
if (on_change)
on_change(*this);
}
void SymField::set_value(std::string_view value) {