From 96f58cbf4baa16b2008888f92d4ce2fb7e615aec Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Tue, 14 May 2013 10:58:46 +1000 Subject: [PATCH] Rework RAOP audio sync commit 9ec7421138adef73ef9cb83b74e046092fecdd6a that fixed audio timestamp calculations revealed an issue in the way RAOP was calculating and setting time syncing. Rework how things are done: We now calculate the audio card latency and simply drop the initial audio frames amounting for such length of time. From that point on, when we get too far behind we drop all queued audio and restart. Rather than dropping a whole audio packet at a time (352 frames) we now drop frame by frame, so re-syncs are much smoother. Surprisingly, this simpler method gives much better results. Synchronisation across multiple airplay device is almost perfect (tested across 2 macs and one linux). Remote audio and video sync are spot on. Fixes #11530 --- .../libmythtv/AirPlay/mythraopconnection.cpp | 149 ++++++++++++------ .../libmythtv/AirPlay/mythraopconnection.h | 2 +- 2 files changed, 98 insertions(+), 53 deletions(-) diff --git a/mythtv/libs/libmythtv/AirPlay/mythraopconnection.cpp b/mythtv/libs/libmythtv/AirPlay/mythraopconnection.cpp index 07f6f144ed0..bdf2777b3b9 100644 --- a/mythtv/libs/libmythtv/AirPlay/mythraopconnection.cpp +++ b/mythtv/libs/libmythtv/AirPlay/mythraopconnection.cpp @@ -33,7 +33,7 @@ QString MythRAOPConnection::g_rsaLastError; // Size (in ms) of audio buffered in audio card -#define AUDIOCARD_BUFFER 800 +#define AUDIOCARD_BUFFER 500 // How frequently we may call ProcessAudio (via QTimer) // ideally 20ms, but according to documentation // anything lower than 50ms on windows, isn't reliable @@ -75,7 +75,7 @@ MythRAOPConnection::MythRAOPConnection(QObject *parent, QTcpSocket *socket, m_lastSequence(0), m_lastTimestamp(0), m_currentTimestamp(0), m_nextSequence(0), m_nextTimestamp(0), m_bufferLength(0), m_timeLastSync(0), - m_cardLatency(0), m_adjustedLatency(0), m_audioStarted(false), + m_cardLatency(-1), m_adjustedLatency(-1), m_audioStarted(false), // clock sync m_masterTimeStamp(0), m_deviceTimeStamp(0), m_networkLatency(0), m_clockSkew(0), @@ -367,49 +367,54 @@ void MythRAOPConnection::ProcessSync(const QByteArray &buf) LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("SYNC: cur:%1 next:%2 time:%3") .arg(m_currentTimestamp).arg(m_nextTimestamp).arg(m_timeLastSync)); - uint64_t delay = framesToMs(m_audioQueue.size() * m_framesPerPacket); - delay += m_networkLatency; + int64_t delay = framesToMs(m_audioQueue.size() * m_framesPerPacket); + int64_t audiots = m_audio->GetAudiotime(); + int64_t currentLatency = 0LL; - // Calculate audio card latency - if (first) - { - m_cardLatency = AudioCardLatency(); - // if audio isn't started, start playing 200ms worth of silence - // and measure timestamp difference - LOG(VB_GENERAL, LOG_DEBUG, LOC + - QString("Audio hardware latency: %1ms").arg(m_cardLatency)); - } - - uint64_t audiots = m_audio->GetAudiotime(); if (m_audioStarted) { - m_adjustedLatency = (int64_t)audiots - (int64_t)m_currentTimestamp; - } - if (m_adjustedLatency > (int64_t)m_bufferLength) - { - // Too much delay in playback - // will reset audio card in next ProcessAudio - m_audioStarted = false; - m_adjustedLatency = 0; + currentLatency = (int64_t)audiots - (int64_t)m_currentTimestamp; } - delay += m_audio->GetAudioBufferedTime(); - delay += m_adjustedLatency; + LOG(VB_GENERAL, LOG_DEBUG, LOC + + QString("RAOP timestamps: about to play:%1 desired:%2 latency:%3") + .arg(audiots).arg(m_currentTimestamp) + .arg(currentLatency)); - // Expire old audio - ExpireResendRequests(m_currentTimestamp); - int res = ExpireAudio(m_currentTimestamp); - if (res > 0) - { - LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("Drop %1 packets").arg(res)); - } + delay += m_audio->GetAudioBufferedTime(); + delay += currentLatency; LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("Queue=%1 buffer=%2ms ideal=%3ms diffts:%4ms") .arg(m_audioQueue.size()) .arg(delay) .arg(m_bufferLength) - .arg(m_adjustedLatency)); + .arg(m_bufferLength-delay)); + + if (m_adjustedLatency <= 0 && m_audioStarted && + (-currentLatency > AUDIOCARD_BUFFER)) + { + // Too much delay in playback. + // The threshold is a value chosen to be loose enough so it doesn't + // trigger too often, but should be low enough to detect any accidental + // interruptions. + // Will drop some frames in next ProcessAudio + LOG(VB_PLAYBACK, LOG_DEBUG, LOC + + QString("Too much delay (%1ms), adjusting") + .arg(m_bufferLength - delay)); + + m_adjustedLatency = m_cardLatency + m_networkLatency; + + // Expire old audio + ExpireResendRequests(m_currentTimestamp - m_adjustedLatency); + int res = ExpireAudio(m_currentTimestamp - m_adjustedLatency); + if (res > 0) + { + LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("Drop %1 packets").arg(res)); + } + + m_audioStarted = false; + } } /** @@ -533,6 +538,8 @@ void MythRAOPConnection::ProcessTimeResponse(const QByteArray &buf) // network latency equal time difference in ms between request and response // divide by two for approximate time of one way trip m_networkLatency = (time2 - time1) / 2; + LOG(VB_AUDIO, LOG_DEBUG, LOC + QString("Network Latency: %1ms") + .arg(m_networkLatency)); // now calculate the time difference between the client and us. // this is NTP time, where sec is in seconds, and ticks is in 1/2^32s @@ -659,7 +666,7 @@ void MythRAOPConnection::ProcessAudio() } timeval t; gettimeofday(&t, NULL); uint64_t dtime = (t.tv_sec * 1000 + t.tv_usec / 1000) - m_timeLastSync; - uint64_t rtp = dtime + m_currentTimestamp + m_networkLatency; + uint64_t rtp = dtime + m_currentTimestamp; uint64_t buffered = m_audioStarted ? m_audio->GetAudioBufferedTime() : 0; // Keep audio framework buffer as short as possible, keeping everything in @@ -669,12 +676,11 @@ void MythRAOPConnection::ProcessAudio() // Also make sure m_audioQueue never goes to less than 1/3 of the RDP stream // total latency, this should gives us enough time to receive missed packets - uint64_t queue = framesToMs(m_audioQueue.size() * m_framesPerPacket); + int64_t queue = framesToMs(m_audioQueue.size() * m_framesPerPacket); if (queue < m_bufferLength / 3) return; rtp += buffered; - rtp += m_cardLatency; // How many packets to add to the audio card, to fill AUDIOCARD_BUFFER int max_packets = ((AUDIOCARD_BUFFER - buffered) @@ -709,8 +715,27 @@ void MythRAOPConnection::ProcessAudio() for (; it != frames.data->end(); ++it) { AudioData *data = &(*it); - m_audio->AddData((char *)data->data, data->length, - timestamp, data->frames); + int offset = 0; + int frames = 0; + + if (m_adjustedLatency > 0) + { + // calculate how many frames we have to drop to catch up + offset = (m_adjustedLatency * m_frameRate / 1000) * + m_audio->GetBytesPerFrame(); + if (offset > data->length) + offset = data->length; + frames = offset / m_audio->GetBytesPerFrame(); + m_adjustedLatency -= framesToMs(frames+1); + LOG(VB_PLAYBACK, LOG_DEBUG, LOC + + QString("ProcessAudio: Dropping %1 frames to catch up " + "(%2ms to go)") + .arg(frames).arg(m_adjustedLatency)); + timestamp += framesToMs(frames); + } + m_audio->AddData((char *)data->data + offset, + data->length - offset, + timestamp, frames); timestamp += m_audio->LengthLastData(); } i++; @@ -930,13 +955,13 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header, if (tags.contains("Apple-Challenge")) { LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("Received Apple-Challenge")); - + *m_textStream << "Apple-Response: "; if (!LoadKey()) return; int tosize = RSA_size(LoadKey()); uint8_t *to = new uint8_t[tosize]; - + QByteArray challenge = QByteArray::fromBase64(tags["Apple-Challenge"].toLatin1()); int challenge_size = challenge.size(); @@ -948,7 +973,7 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header, if (challenge_size > 16) challenge_size = 16; } - + int i = 0; unsigned char from[38]; memcpy(from, challenge.constData(), challenge_size); @@ -980,24 +1005,24 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header, } memcpy(from + i, m_hardwareId.constData(), AIRPLAY_HARDWARE_ID_SIZE); i += AIRPLAY_HARDWARE_ID_SIZE; - + int pad = 32 - i; if (pad > 0) { memset(from + i, 0, pad); i += pad; } - + LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("Full base64 response: '%1' size %2") .arg(QByteArray((const char *)from, i).toBase64().constData()) .arg(i)); - + RSA_private_encrypt(i, from, to, LoadKey(), RSA_PKCS1_PADDING); - + QByteArray base64 = QByteArray((const char *)to, tosize).toBase64(); delete[] to; - + for (int pos = base64.size() - 1; pos > 0; pos--) { if (base64[pos] == '=') @@ -1205,7 +1230,19 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header, } if (OpenAudioDevice()) + { CreateDecoder(); + // Calculate audio card latency + if (m_cardLatency < 0) + { + m_adjustedLatency = m_cardLatency = AudioCardLatency(); + // if audio isn't started, start playing 500ms worth of silence + // and measure timestamp difference + LOG(VB_GENERAL, LOG_DEBUG, LOC + + QString("Audio hardware latency: %1ms") + .arg(m_cardLatency + m_networkLatency)); + } + } // Recreate transport line with new ports value QString newdata; @@ -1243,6 +1280,9 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header, *m_textStream << "Transport: " << newdata << "\r\n"; *m_textStream << "Session: 1\r\n"; *m_textStream << "Audio-Jack-Status: connected\r\n"; + + // Ask for master clock value to determine time skew and average network latency + SendTimeRequest(); } else { @@ -1257,8 +1297,13 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header, m_nextSequence = RTPseq; m_nextTimestamp = RTPtimestamp; } - // Ask for master clock value to determine time skew and average network latency - SendTimeRequest(); + + // Calculate audio card latency + if (m_cardLatency > 0) + { + *m_textStream << QString("Audio-Latency: %1") + .arg(m_cardLatency+m_networkLatency); + } } else if (option == "FLUSH") { @@ -1660,18 +1705,18 @@ int64_t MythRAOPConnection::AudioCardLatency(void) if (!m_audio) return 0; - uint64_t timestamp = 123456; - int16_t *samples = (int16_t *)av_mallocz(AVCODEC_MAX_AUDIO_FRAME_SIZE); int frames = AUDIOCARD_BUFFER * m_frameRate / 1000; m_audio->AddData((char *)samples, frames * (m_sampleSize>>3) * m_channels, - timestamp, + 0, frames); av_free(samples); usleep(AUDIOCARD_BUFFER * 1000); uint64_t audiots = m_audio->GetAudiotime(); - return (int64_t)timestamp - (int64_t)audiots; + LOG(VB_PLAYBACK, LOG_DEBUG, LOC + QString("AudioCardLatency: ts=%1ms") + .arg(audiots)); + return AUDIOCARD_BUFFER - (int64_t)audiots; } void MythRAOPConnection::newEventClient(QTcpSocket *client) diff --git a/mythtv/libs/libmythtv/AirPlay/mythraopconnection.h b/mythtv/libs/libmythtv/AirPlay/mythraopconnection.h index 38162afcd0b..23fd2c15656 100644 --- a/mythtv/libs/libmythtv/AirPlay/mythraopconnection.h +++ b/mythtv/libs/libmythtv/AirPlay/mythraopconnection.h @@ -157,7 +157,7 @@ class MTV_PUBLIC MythRAOPConnection : public QObject uint64_t m_currentTimestamp; uint16_t m_nextSequence; uint64_t m_nextTimestamp; - uint64_t m_bufferLength; + int64_t m_bufferLength; uint64_t m_timeLastSync; int64_t m_cardLatency; int64_t m_adjustedLatency;