POCSAG Processor Rewrite (#1437)

* WIP Refactoring
* WordExtractor building
* Fix buffer sizes and squelch execute
* Move impls to cpp file
* Baud indicator
* WIP new bit extractor
* New approach for bit extraction.
* Code fit and finish
* Fix case on button
* Cleanup
* Adjust rate miss threshold
* Fix count bits error calculation.
This commit is contained in:
Kyle Reed 2023-09-08 10:41:09 -07:00 committed by GitHub
parent 9525738118
commit 31e8019642
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 648 additions and 534 deletions

View file

@ -3,7 +3,7 @@
* Copyright (C) 2012-2014 Elias Oenal (multimon-ng@eliasoenal.com)
* Copyright (C) 2015 Jared Boone, ShareBrained Technology, Inc.
* Copyright (C) 2016 Furrtek
* Copyright (C) 2016 Kyle Reed
* Copyright (C) 2023 Kyle Reed
*
* This file is part of PortaPack.
*
@ -34,6 +34,323 @@
using namespace std;
namespace {
/* Count of bits that differ between the two values. */
uint8_t differ_bit_count(uint32_t left, uint32_t right) {
uint32_t diff = left ^ right;
uint8_t count = 0;
for (size_t i = 0; i < sizeof(diff) * 8; ++i) {
if (((diff >> i) & 0x1) == 1)
++count;
}
return count;
}
} // namespace
/* AudioNormalizer ***************************************/
void AudioNormalizer::execute_in_place(const buffer_f32_t& audio) {
// Decay min/max every second (@24kHz).
if (counter_ >= 24'000) {
// 90% decay factor seems to work well.
// This keeps large transients from wrecking the filter.
max_ *= 0.9f;
min_ *= 0.9f;
counter_ = 0;
calculate_thresholds();
}
counter_ += audio.count;
for (size_t i = 0; i < audio.count; ++i) {
auto& val = audio.p[i];
if (val > max_) {
max_ = val;
calculate_thresholds();
}
if (val < min_) {
min_ = val;
calculate_thresholds();
}
if (val >= t_hi_)
val = 1.0f;
else if (val <= t_lo_)
val = -1.0f;
else
val = 0.0;
}
}
void AudioNormalizer::calculate_thresholds() {
auto center = (max_ + min_) / 2.0f;
auto range = (max_ - min_) / 2.0f;
// 10% off center force either +/-1.0f.
// Higher == larger dead zone.
// Lower == more false positives.
auto threshold = range * 0.1;
t_hi_ = center + threshold;
t_lo_ = center - threshold;
}
/* BitQueue **********************************************/
void BitQueue::push(bool bit) {
data_ = (data_ << 1) | (bit ? 1 : 0);
if (count_ < max_size_) ++count_;
}
bool BitQueue::pop() {
if (count_ == 0) return false;
--count_;
return (data_ & (1 << count_)) != 0;
}
void BitQueue::reset() {
data_ = 0;
count_ = 0;
}
uint8_t BitQueue::size() const {
return count_;
}
uint32_t BitQueue::data() const {
return data_;
}
/* BitExtractor ******************************************/
void BitExtractor::extract_bits(const buffer_f32_t& audio) {
// Assumes input has been normalized +/- 1.0f.
for (size_t i = 0; i < audio.count; ++i) {
sample_ = audio.p[i];
++sample_index_;
// There's a transition when both sides of the XOR are the
// same which will result in a the overall value being 0.
bool is_transition = ((last_sample_ < 0) ^ (sample_ >= 0)) == 0;
if (is_transition) {
if (handle_transition())
bad_transitions_ = 0;
else
++bad_transitions_;
// Too many bad transitions? Reset.
if (bad_transitions_ > bad_transition_reset_threshold)
reset();
}
// Time to push the next bit?
if (sample_index_ >= next_bit_center_) {
// Use the two most recent samples for the bit value.
auto val = (sample_ + last_sample_) / 2.0;
bits_.push(val < 0); // NB: '1' is negative.
if (current_rate_)
next_bit_center_ += current_rate_->bit_length;
}
last_sample_ = sample_;
}
}
void BitExtractor::configure(uint32_t sample_rate) {
sample_rate_ = sample_rate;
min_valid_length_ = UINT16_MAX;
// Build the baud rate info table based on the sample rate.
for (auto& info : known_rates_) {
info.bit_length = sample_rate / info.baud_rate;
// Allow for 20% deviation.
info.min_bit_length = 0.80 * info.bit_length;
info.max_bit_length = 1.20 * info.bit_length;
if (info.min_bit_length < min_valid_length_)
min_valid_length_ = info.min_bit_length;
}
reset();
}
void BitExtractor::reset() {
current_rate_ = nullptr;
rate_misses_ = 0;
sample_ = 0.0;
last_sample_ = 0.0;
next_bit_center_ = 0.0;
sample_index_ = 0;
last_transition_index_ = 0;
bad_transitions_ = 0;
}
uint16_t BitExtractor::baud_rate() const {
return current_rate_ ? current_rate_->baud_rate : 0;
}
bool BitExtractor::handle_transition() {
auto length = sample_index_ - last_transition_index_;
last_transition_index_ = sample_index_;
// Length is too short, ignore this.
if (length <= min_valid_length_) return false;
// TODO: should the following be "bad" or "rate misses"?
// Is length a multiple of the current rate's bit length?
uint16_t bit_count = 0;
if (!count_bits(length, bit_count)) return false;
// Does the bit length correspond to a known rate?
auto bit_length = length / static_cast<float>(bit_count);
auto rate = get_baud_info(bit_length);
if (!rate) return false;
// Set current rate if it hasn't been set yet.
if (!current_rate_)
current_rate_ = rate;
// Maybe current rate isn't the best rate?
auto rate_miss = rate != current_rate_;
if (rate_miss) {
++rate_misses_;
// Lots of rate misses, try another rate.
if (rate_misses_ > rate_miss_reset_threshold) {
current_rate_ = rate;
rate_misses_ = 0;
}
} else {
// Transition is aligned with the current rate, predict next bit.
auto half_bit = current_rate_->bit_length / 2.0;
next_bit_center_ = sample_index_ + half_bit;
}
return true;
}
bool BitExtractor::count_bits(uint32_t length, uint16_t& bit_count) {
bit_count = 0;
// No rate yet, assume one valid bit. Downstream will deal with it.
if (!current_rate_) {
bit_count = 1;
return true;
}
// How many bits span the specified length?
float exact_bits = length / current_rate_->bit_length;
// < 1 bit, current rate is probably too low.
if (exact_bits < 0.80) return false;
// Round to the nearest # of bits and determine how
// well the current rate fits the data.
float round_bits = std::round(exact_bits);
float error = std::abs(exact_bits - round_bits) / exact_bits;
// Good transition are w/in 15% of current rate estimate.
bit_count = round_bits;
return error < 0.15;
}
const BitExtractor::BaudInfo* BitExtractor::get_baud_info(float bit_length) const {
// NB: This assumes known_rates_ are ordered slowest first.
for (const auto& info : known_rates_) {
if (bit_length >= info.min_bit_length &&
bit_length <= info.max_bit_length) {
return &info;
}
}
return nullptr;
}
/* CodewordExtractor *************************************/
void CodewordExtractor::process_bits() {
// Process all of the bits in the bits queue.
while (bits_.size() > 0) {
take_one_bit();
// Wait until data_ is full.
if (bit_count_ < data_bit_count)
continue;
// Wait for the sync frame.
if (!has_sync_) {
if (differ_bit_count(data_, sync_codeword) <= 2)
handle_sync(/*inverted=*/false);
else if (differ_bit_count(data_, ~sync_codeword) <= 2)
handle_sync(/*inverted=*/true);
continue;
}
save_current_codeword();
if (word_count_ == pocsag::batch_size)
handle_batch_complete();
}
}
void CodewordExtractor::flush() {
// Don't bother flushing if there's no pending data.
if (word_count_ == 0) return;
pad_idle();
handle_batch_complete();
}
void CodewordExtractor::reset() {
clear_data_bits();
has_sync_ = false;
inverted_ = false;
word_count_ = 0;
}
void CodewordExtractor::clear_data_bits() {
data_ = 0;
bit_count_ = 0;
}
void CodewordExtractor::take_one_bit() {
data_ = (data_ << 1) | bits_.pop();
if (bit_count_ < data_bit_count)
++bit_count_;
}
void CodewordExtractor::handle_sync(bool inverted) {
clear_data_bits();
has_sync_ = true;
inverted_ = inverted;
word_count_ = 0;
}
void CodewordExtractor::save_current_codeword() {
batch_[word_count_++] = inverted_ ? ~data_ : data_;
clear_data_bits();
}
void CodewordExtractor::handle_batch_complete() {
on_batch_(*this);
has_sync_ = false;
word_count_ = 0;
}
void CodewordExtractor::pad_idle() {
while (word_count_ < pocsag::batch_size)
batch_[word_count_++] = idle_codeword;
}
/* POCSAGProcessor ***************************************/
void POCSAGProcessor::execute(const buffer_c8_t& buffer) {
if (!configured) return;
@ -52,12 +369,12 @@ void POCSAGProcessor::execute(const buffer_c8_t& buffer) {
bool has_audio = squelch.execute(audio);
squelch_history = (squelch_history << 1) | (has_audio ? 1 : 0);
// Has there been any signal?
// Has there been any signal recently?
if (squelch_history == 0) {
// No signal for a while, flush and reset.
if (!has_been_reset) {
OnDataFrame(m_numCode, getRate());
resetVals();
// No recent signal, flush and prepare for next message.
if (word_extractor.current() > 0) {
flush();
reset();
send_stats();
}
@ -69,18 +386,20 @@ void POCSAGProcessor::execute(const buffer_c8_t& buffer) {
return;
}
// Filter out high-frequency noise. TODO: compensate gain?
// Filter out high-frequency noise then normalize.
lpf.execute_in_place(audio);
normalizer.execute_in_place(audio);
audio_output.write(audio);
processDemodulatedSamples(audio.p, 16);
extractFrames();
// Decode the messages from the audio.
bit_extractor.extract_bits(audio);
word_extractor.process_bits();
// Update the status.
samples_processed += buffer.count;
if (samples_processed >= stat_update_threshold) {
send_stats();
samples_processed = 0;
samples_processed -= stat_update_threshold;
}
}
@ -104,8 +423,8 @@ void POCSAGProcessor::on_message(const Message* const message) {
void POCSAGProcessor::configure() {
constexpr size_t decim_0_output_fs = baseband_fs / decim_0.decimation_factor;
constexpr size_t decim_1_output_fs = decim_0_output_fs / decim_1.decimation_factor;
const size_t channel_filter_output_fs = decim_1_output_fs / 2;
const size_t demod_input_fs = channel_filter_output_fs;
constexpr size_t channel_filter_output_fs = decim_1_output_fs / 2;
constexpr size_t demod_input_fs = channel_filter_output_fs;
decim_0.configure(taps_11k0_decim_0.taps);
decim_1.configure(taps_11k0_decim_1.taps);
@ -115,384 +434,41 @@ void POCSAGProcessor::configure() {
// Don't process the audio stream.
audio_output.configure(false);
// Set up the frame extraction, limits of baud.
setFrameExtractParams(demod_input_fs, 4000, 300, 32);
bit_extractor.configure(demod_input_fs);
// Set ready to process data.
configured = true;
}
void POCSAGProcessor::send_stats() const {
POCSAGStatsMessage message(m_fifo.codeword, m_numCode, m_gotSync);
shared_memory.application_queue.push(message);
void POCSAGProcessor::flush() {
word_extractor.flush();
}
int POCSAGProcessor::OnDataWord(uint32_t word, int pos) {
packet.set(pos, word);
return 0;
}
int POCSAGProcessor::OnDataFrame(int len, int baud) {
if (len > 0) {
packet.set_bitrate(baud);
packet.set_flag(pocsag::PacketFlag::NORMAL);
packet.set_timestamp(Timestamp::now());
const POCSAGPacketMessage message(packet);
shared_memory.application_queue.push(message);
}
return 0;
}
#define BAUD_STABLE (104)
#define MAX_CONSEC_SAME (32)
#define MAX_WITHOUT_SINGLE (64)
#define MAX_BAD_TRANS (10)
#define M_SYNC (0x7cd215d8)
#define M_NOTSYNC (0x832dea27)
#define M_IDLE (0x7a89c197)
inline int bitsDiff(unsigned long left, unsigned long right) {
unsigned long xord = left ^ right;
int count = 0;
for (int i = 0; i < 32; i++) {
if ((xord & 0x01) != 0) ++count;
xord = xord >> 1;
}
return (count);
}
void POCSAGProcessor::initFrameExtraction() {
m_averageSymbolLen_1024 = m_maxSymSamples_1024;
m_lastStableSymbolLen_1024 = m_minSymSamples_1024;
m_badTransitions = 0;
m_bitsStart = 0;
m_bitsEnd = 0;
m_inverted = false;
resetVals();
}
void POCSAGProcessor::resetVals() {
if (has_been_reset) return;
// Reset the parameters
m_goodTransitions = 0;
m_badTransitions = 0;
m_averageSymbolLen_1024 = m_maxSymSamples_1024;
m_shortestGoodTrans_1024 = m_maxSymSamples_1024;
m_valMid = 0;
// And reset the counts
m_lastTransPos_1024 = 0;
m_lastBitPos_1024 = 0;
m_lastSample = 0;
m_sampleNo = 0;
m_nextBitPos_1024 = m_maxSymSamples_1024;
m_nextBitPosInt = (long)m_nextBitPos_1024;
// Extraction
m_fifo.numBits = 0;
m_fifo.codeword = 0;
m_gotSync = false;
m_numCode = 0;
has_been_reset = true;
void POCSAGProcessor::reset() {
bits.reset();
bit_extractor.reset();
word_extractor.reset();
samples_processed = 0;
}
void POCSAGProcessor::setFrameExtractParams(long a_samplesPerSec, long a_maxBaud, long a_minBaud, long maxRunOfSameValue) {
m_samplesPerSec = a_samplesPerSec;
m_minSymSamples_1024 = (uint32_t)(1024.0f * (float)a_samplesPerSec / (float)a_maxBaud);
m_maxSymSamples_1024 = (uint32_t)(1024.0f * (float)a_samplesPerSec / (float)a_minBaud);
m_maxRunOfSameValue = maxRunOfSameValue;
m_shortestGoodTrans_1024 = m_maxSymSamples_1024;
m_averageSymbolLen_1024 = m_maxSymSamples_1024;
m_lastStableSymbolLen_1024 = m_minSymSamples_1024;
m_nextBitPos_1024 = m_averageSymbolLen_1024 / 2;
m_nextBitPosInt = m_nextBitPos_1024 >> 10;
initFrameExtraction();
void POCSAGProcessor::send_stats() const {
POCSAGStatsMessage message(
word_extractor.current(), word_extractor.count(),
word_extractor.has_sync(), bit_extractor.baud_rate());
shared_memory.application_queue.push(message);
}
int POCSAGProcessor::processDemodulatedSamples(float* sampleBuff, int noOfSamples) {
bool transition = false;
uint32_t samplePos_1024 = 0;
uint32_t len_1024 = 0;
void POCSAGProcessor::send_packet() {
packet.set_flag(pocsag::PacketFlag::NORMAL);
packet.set_timestamp(Timestamp::now());
packet.set_bitrate(bit_extractor.baud_rate());
packet.set(word_extractor.batch());
has_been_reset = false;
// Loop through the block of data
// ------------------------------
for (int pos = 0; pos < noOfSamples; ++pos) {
m_sample = sampleBuff[pos];
m_valMid += (m_sample - m_valMid) / 1024.0f;
++m_sampleNo;
// Detect Transition
// -----------------
transition = !((m_lastSample < m_valMid) ^ (m_sample >= m_valMid)); // use XOR for speed
// If this is a transition
// -----------------------
if (transition) {
// Calculate samples since last trans
// ----------------------------------
int32_t fractional_1024 = (int32_t)(((m_sample - m_valMid) * 1024) / (m_sample - m_lastSample));
if (fractional_1024 < 0) {
fractional_1024 = -fractional_1024;
}
samplePos_1024 = (m_sampleNo << 10) - fractional_1024;
len_1024 = samplePos_1024 - m_lastTransPos_1024;
m_lastTransPos_1024 = samplePos_1024;
// If symbol is large enough to be valid
// -------------------------------------
if (len_1024 > m_minSymSamples_1024) {
// Check for shortest good transition
// ----------------------------------
if ((len_1024 < m_shortestGoodTrans_1024) &&
(m_goodTransitions < BAUD_STABLE)) // detect change of symbol size
{
int32_t fractionOfShortest_1024 = (len_1024 << 10) / m_shortestGoodTrans_1024;
// If currently at half the baud rate
// ----------------------------------
if ((fractionOfShortest_1024 > 410) && (fractionOfShortest_1024 < 614)) // 0.4 and 0.6
{
m_averageSymbolLen_1024 /= 2;
m_shortestGoodTrans_1024 = len_1024;
}
// If currently at the wrong baud rate
// -----------------------------------
else if (fractionOfShortest_1024 < 768) // 0.75
{
m_averageSymbolLen_1024 = len_1024;
m_shortestGoodTrans_1024 = len_1024;
m_goodTransitions = 0;
m_lastSingleBitPos_1024 = samplePos_1024 - len_1024;
}
}
// Calc the number of bits since events
// ------------------------------------
int32_t halfSymbol_1024 = m_averageSymbolLen_1024 / 2;
int bitsSinceLastTrans = max((uint32_t)1, (len_1024 + halfSymbol_1024) / m_averageSymbolLen_1024);
int bitsSinceLastSingle = (((m_sampleNo << 10) - m_lastSingleBitPos_1024) + halfSymbol_1024) / m_averageSymbolLen_1024;
// Check for single bit
// --------------------
if (bitsSinceLastTrans == 1) {
m_lastSingleBitPos_1024 = samplePos_1024;
}
// If too long since last transition
// ---------------------------------
if (bitsSinceLastTrans > MAX_CONSEC_SAME) {
resetVals();
}
// If too long sice last single bit
// --------------------------------
else if (bitsSinceLastSingle > MAX_WITHOUT_SINGLE) {
resetVals();
} else {
// If this is a good transition
// ----------------------------
int32_t offsetFromExtectedTransition_1024 = len_1024 - (bitsSinceLastTrans * m_averageSymbolLen_1024);
if (offsetFromExtectedTransition_1024 < 0) {
offsetFromExtectedTransition_1024 = -offsetFromExtectedTransition_1024;
}
if (offsetFromExtectedTransition_1024 < ((int32_t)m_averageSymbolLen_1024 / 4)) // Has to be within 1/4 of symbol to be good
{
++m_goodTransitions;
uint32_t bitsCount = min((uint32_t)BAUD_STABLE, m_goodTransitions);
uint32_t propFromPrevious = m_averageSymbolLen_1024 * bitsCount;
uint32_t propFromCurrent = (len_1024 / bitsSinceLastTrans);
m_averageSymbolLen_1024 = (propFromPrevious + propFromCurrent) / (bitsCount + 1);
m_badTransitions = 0;
// if ( len < m_shortestGoodTrans ){m_shortestGoodTrans = len;}
// Store the old symbol size
if (m_goodTransitions >= BAUD_STABLE) {
m_lastStableSymbolLen_1024 = m_averageSymbolLen_1024;
}
}
}
// Set the point of the last bit if not yet stable
// -----------------------------------------------
if ((m_goodTransitions < BAUD_STABLE) || (m_badTransitions > 0)) {
m_lastBitPos_1024 = samplePos_1024 - (m_averageSymbolLen_1024 / 2);
}
// Calculate the exact positiom of the next bit
// --------------------------------------------
int32_t thisPlusHalfsymbol_1024 = samplePos_1024 + (m_averageSymbolLen_1024 / 2);
int32_t lastPlusSymbol = m_lastBitPos_1024 + m_averageSymbolLen_1024;
m_nextBitPos_1024 = lastPlusSymbol + ((thisPlusHalfsymbol_1024 - lastPlusSymbol) / 16);
// Check for bad pos error
// -----------------------
if (m_nextBitPos_1024 < samplePos_1024) m_nextBitPos_1024 += m_averageSymbolLen_1024;
// Calculate integer sample after next bit
// ---------------------------------------
m_nextBitPosInt = (m_nextBitPos_1024 >> 10) + 1;
} // symbol is large enough to be valid
else {
// Bad transition, so reset the counts
// -----------------------------------
++m_badTransitions;
if (m_badTransitions > MAX_BAD_TRANS) {
resetVals();
}
}
} // end of if transition
// Reached the point of the next bit
// ---------------------------------
if (m_sampleNo >= m_nextBitPosInt) {
// Everything is good so extract a bit
// -----------------------------------
if (m_goodTransitions > 20) {
// Store value at the center of bit
// --------------------------------
storeBit();
}
// Check for long 1 or zero
// ------------------------
uint32_t bitsSinceLastTrans = ((m_sampleNo << 10) - m_lastTransPos_1024) / m_averageSymbolLen_1024;
if (bitsSinceLastTrans > m_maxRunOfSameValue) {
resetVals();
}
// Store the point of the last bit
// -------------------------------
m_lastBitPos_1024 = m_nextBitPos_1024;
// Calculate the exact point of the next bit
// -----------------------------------------
m_nextBitPos_1024 += m_averageSymbolLen_1024;
// Look for the bit after the next bit pos
// ---------------------------------------
m_nextBitPosInt = (m_nextBitPos_1024 >> 10) + 1;
} // Reached the point of the next bit
m_lastSample = m_sample;
} // Loop through the block of data
return getNoOfBits();
POCSAGPacketMessage message(packet);
shared_memory.application_queue.push(message);
}
void POCSAGProcessor::storeBit() {
if (++m_bitsStart >= BIT_BUF_SIZE) {
m_bitsStart = 0;
}
// Calculate the bit value
float sample = (m_sample + m_lastSample) / 2;
// int32_t sample_1024 = m_sample_1024;
bool bit = sample > m_valMid;
// If buffer not full
if (m_bitsStart != m_bitsEnd) {
// Decide on output val
if (bit) {
m_bits[m_bitsStart] = 0;
} else {
m_bits[m_bitsStart] = 1;
}
}
// Throw away bits if the buffer is full
else {
if (--m_bitsStart <= -1) {
m_bitsStart = BIT_BUF_SIZE - 1;
}
}
}
int POCSAGProcessor::extractFrames() {
int msgCnt = 0;
// While there is unread data in the bits buffer
//----------------------------------------------
while (getNoOfBits() > 0) {
m_fifo.codeword = (m_fifo.codeword << 1) + getBit();
m_fifo.numBits++;
// If number of bits in fifo equals 32
//------------------------------------
if (m_fifo.numBits >= 32) {
// Not got sync
// ------------
if (!m_gotSync) {
if (bitsDiff(m_fifo.codeword, M_SYNC) <= 2) {
m_inverted = false;
m_gotSync = true;
m_numCode = -1;
m_fifo.numBits = 0;
} else if (bitsDiff(m_fifo.codeword, M_NOTSYNC) <= 2) {
m_inverted = true;
m_gotSync = true;
m_numCode = -1;
m_fifo.numBits = 0;
} else {
// Cause it to load one more bit
m_fifo.numBits = 31;
}
} // Not got sync
else {
// Increment the word count
// ------------------------
++m_numCode; // It got set to -1 when a sync was found, now count the 16 words
uint32_t val = m_inverted ? ~m_fifo.codeword : m_fifo.codeword;
OnDataWord(val, m_numCode);
// If at the end of a 16 word block
// --------------------------------
if (m_numCode >= 15) {
msgCnt += OnDataFrame(m_numCode + 1, getRate());
m_gotSync = false;
m_numCode = -1;
}
m_fifo.numBits = 0;
}
} // If number of bits in fifo equals 32
} // While there is unread data in the bits buffer
return msgCnt;
} // extractFrames
short POCSAGProcessor::getBit() {
if (m_bitsEnd != m_bitsStart) {
if (++m_bitsEnd >= BIT_BUF_SIZE) {
m_bitsEnd = 0;
}
return m_bits[m_bitsEnd];
} else {
return -1;
}
}
int POCSAGProcessor::getNoOfBits() {
int bits = m_bitsEnd - m_bitsStart;
if (bits < 0) {
bits += BIT_BUF_SIZE;
}
return bits;
}
uint32_t POCSAGProcessor::getRate() {
return ((m_samplesPerSec << 10) + 512) / m_lastStableSymbolLen_1024;
}
/* main **************************************************/
int main() {
EventDispatcher event_dispatcher{std::make_unique<POCSAGProcessor>()};