Skip to content

Commit

Permalink
Rework RAOP audio sync
Browse files Browse the repository at this point in the history 
commit 9ec7421 that fixed audio timestamp calculations revealed an issue in the way RAOP was calculating and setting time syncing.
Rework how things are done: We now calculate the audio card latency and simply drop the initial audio frames amounting for such length of time. From that point on, when we get too far behind we drop all queued audio and restart.

Rather than dropping a whole audio packet at a time (352 frames) we now drop frame by frame, so re-syncs are much smoother.

Surprisingly, this simpler method gives much better results. Synchronisation across multiple airplay device is almost perfect (tested across 2 macs and one linux). Remote audio and video sync are spot on.

Fixes #11530
  • Loading branch information
@jyavenard
jyavenard committed May 14, 2013
1 parent b3097d6 commit 96f58cb
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 53 deletions.
149 changes: 97 additions & 52 deletions mythtv/libs/libmythtv/AirPlay/mythraopconnection.cpp
View file
Expand Up @@ -33,7 +33,7 @@ QString MythRAOPConnection::g_rsaLastError;


// Size (in ms) of audio buffered in audio card
#define AUDIOCARD_BUFFER 800
#define AUDIOCARD_BUFFER 500
// How frequently we may call ProcessAudio (via QTimer)
// ideally 20ms, but according to documentation
// anything lower than 50ms on windows, isn't reliable
Expand Down Expand Up @@ -75,7 +75,7 @@ MythRAOPConnection::MythRAOPConnection(QObject *parent, QTcpSocket *socket,
m_lastSequence(0), m_lastTimestamp(0),
m_currentTimestamp(0), m_nextSequence(0), m_nextTimestamp(0),
m_bufferLength(0), m_timeLastSync(0),
m_cardLatency(0), m_adjustedLatency(0), m_audioStarted(false),
m_cardLatency(-1), m_adjustedLatency(-1), m_audioStarted(false),
// clock sync
m_masterTimeStamp(0), m_deviceTimeStamp(0), m_networkLatency(0),
m_clockSkew(0),
Expand Down Expand Up @@ -367,49 +367,54 @@ void MythRAOPConnection::ProcessSync(const QByteArray &buf)
LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("SYNC: cur:%1 next:%2 time:%3")
.arg(m_currentTimestamp).arg(m_nextTimestamp).arg(m_timeLastSync));

uint64_t delay = framesToMs(m_audioQueue.size() * m_framesPerPacket);
delay += m_networkLatency;
int64_t delay = framesToMs(m_audioQueue.size() * m_framesPerPacket);
int64_t audiots = m_audio->GetAudiotime();
int64_t currentLatency = 0LL;

// Calculate audio card latency
if (first)
{
m_cardLatency = AudioCardLatency();
// if audio isn't started, start playing 200ms worth of silence
// and measure timestamp difference
LOG(VB_GENERAL, LOG_DEBUG, LOC +
QString("Audio hardware latency: %1ms").arg(m_cardLatency));
}

uint64_t audiots = m_audio->GetAudiotime();
if (m_audioStarted)
{
m_adjustedLatency = (int64_t)audiots - (int64_t)m_currentTimestamp;
}
if (m_adjustedLatency > (int64_t)m_bufferLength)
{
// Too much delay in playback
// will reset audio card in next ProcessAudio
m_audioStarted = false;
m_adjustedLatency = 0;
currentLatency = (int64_t)audiots - (int64_t)m_currentTimestamp;
}

delay += m_audio->GetAudioBufferedTime();
delay += m_adjustedLatency;
LOG(VB_GENERAL, LOG_DEBUG, LOC +
QString("RAOP timestamps: about to play:%1 desired:%2 latency:%3")
.arg(audiots).arg(m_currentTimestamp)
.arg(currentLatency));

// Expire old audio
ExpireResendRequests(m_currentTimestamp);
int res = ExpireAudio(m_currentTimestamp);
if (res > 0)
{
LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("Drop %1 packets").arg(res));
}
delay += m_audio->GetAudioBufferedTime();
delay += currentLatency;

LOG(VB_GENERAL, LOG_DEBUG, LOC +
QString("Queue=%1 buffer=%2ms ideal=%3ms diffts:%4ms")
.arg(m_audioQueue.size())
.arg(delay)
.arg(m_bufferLength)
.arg(m_adjustedLatency));
.arg(m_bufferLength-delay));

if (m_adjustedLatency <= 0 && m_audioStarted &&
(-currentLatency > AUDIOCARD_BUFFER))
{
// Too much delay in playback.
// The threshold is a value chosen to be loose enough so it doesn't
// trigger too often, but should be low enough to detect any accidental
// interruptions.
// Will drop some frames in next ProcessAudio
LOG(VB_PLAYBACK, LOG_DEBUG, LOC +
QString("Too much delay (%1ms), adjusting")
.arg(m_bufferLength - delay));

m_adjustedLatency = m_cardLatency + m_networkLatency;

// Expire old audio
ExpireResendRequests(m_currentTimestamp - m_adjustedLatency);
int res = ExpireAudio(m_currentTimestamp - m_adjustedLatency);
if (res > 0)
{
LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("Drop %1 packets").arg(res));
}

m_audioStarted = false;
}
}

/**
Expand Down Expand Up @@ -533,6 +538,8 @@ void MythRAOPConnection::ProcessTimeResponse(const QByteArray &buf)
// network latency equal time difference in ms between request and response
// divide by two for approximate time of one way trip
m_networkLatency = (time2 - time1) / 2;
LOG(VB_AUDIO, LOG_DEBUG, LOC + QString("Network Latency: %1ms")
.arg(m_networkLatency));

// now calculate the time difference between the client and us.
// this is NTP time, where sec is in seconds, and ticks is in 1/2^32s
Expand Down Expand Up @@ -659,7 +666,7 @@ void MythRAOPConnection::ProcessAudio()
}
timeval t; gettimeofday(&t, NULL);
uint64_t dtime = (t.tv_sec * 1000 + t.tv_usec / 1000) - m_timeLastSync;
uint64_t rtp = dtime + m_currentTimestamp + m_networkLatency;
uint64_t rtp = dtime + m_currentTimestamp;
uint64_t buffered = m_audioStarted ? m_audio->GetAudioBufferedTime() : 0;

// Keep audio framework buffer as short as possible, keeping everything in
Expand All @@ -669,12 +676,11 @@ void MythRAOPConnection::ProcessAudio()

// Also make sure m_audioQueue never goes to less than 1/3 of the RDP stream
// total latency, this should gives us enough time to receive missed packets
uint64_t queue = framesToMs(m_audioQueue.size() * m_framesPerPacket);
int64_t queue = framesToMs(m_audioQueue.size() * m_framesPerPacket);
if (queue < m_bufferLength / 3)
return;

rtp += buffered;
rtp += m_cardLatency;

// How many packets to add to the audio card, to fill AUDIOCARD_BUFFER
int max_packets = ((AUDIOCARD_BUFFER - buffered)
Expand Down Expand Up @@ -709,8 +715,27 @@ void MythRAOPConnection::ProcessAudio()
for (; it != frames.data->end(); ++it)
{
AudioData *data = &(*it);
m_audio->AddData((char *)data->data, data->length,
timestamp, data->frames);
int offset = 0;
int frames = 0;

if (m_adjustedLatency > 0)
{
// calculate how many frames we have to drop to catch up
offset = (m_adjustedLatency * m_frameRate / 1000) *
m_audio->GetBytesPerFrame();
if (offset > data->length)
offset = data->length;
frames = offset / m_audio->GetBytesPerFrame();
m_adjustedLatency -= framesToMs(frames+1);
LOG(VB_PLAYBACK, LOG_DEBUG, LOC +
QString("ProcessAudio: Dropping %1 frames to catch up "
"(%2ms to go)")
.arg(frames).arg(m_adjustedLatency));
timestamp += framesToMs(frames);
}
m_audio->AddData((char *)data->data + offset,
data->length - offset,
timestamp, frames);
timestamp += m_audio->LengthLastData();
}
i++;
Expand Down Expand Up @@ -930,13 +955,13 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header,
if (tags.contains("Apple-Challenge"))
{
LOG(VB_GENERAL, LOG_DEBUG, LOC + QString("Received Apple-Challenge"));

*m_textStream << "Apple-Response: ";
if (!LoadKey())
return;
int tosize = RSA_size(LoadKey());
uint8_t *to = new uint8_t[tosize];

QByteArray challenge =
QByteArray::fromBase64(tags["Apple-Challenge"].toLatin1());
int challenge_size = challenge.size();
Expand All @@ -948,7 +973,7 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header,
if (challenge_size > 16)
challenge_size = 16;
}

int i = 0;
unsigned char from[38];
memcpy(from, challenge.constData(), challenge_size);
Expand Down Expand Up @@ -980,24 +1005,24 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header,
}
memcpy(from + i, m_hardwareId.constData(), AIRPLAY_HARDWARE_ID_SIZE);
i += AIRPLAY_HARDWARE_ID_SIZE;

int pad = 32 - i;
if (pad > 0)
{
memset(from + i, 0, pad);
i += pad;
}

LOG(VB_GENERAL, LOG_DEBUG, LOC +
QString("Full base64 response: '%1' size %2")
.arg(QByteArray((const char *)from, i).toBase64().constData())
.arg(i));

RSA_private_encrypt(i, from, to, LoadKey(), RSA_PKCS1_PADDING);

QByteArray base64 = QByteArray((const char *)to, tosize).toBase64();
delete[] to;

for (int pos = base64.size() - 1; pos > 0; pos--)
{
if (base64[pos] == '=')
Expand Down Expand Up @@ -1205,7 +1230,19 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header,
}

if (OpenAudioDevice())
{
CreateDecoder();
// Calculate audio card latency
if (m_cardLatency < 0)
{
m_adjustedLatency = m_cardLatency = AudioCardLatency();
// if audio isn't started, start playing 500ms worth of silence
// and measure timestamp difference
LOG(VB_GENERAL, LOG_DEBUG, LOC +
QString("Audio hardware latency: %1ms")
.arg(m_cardLatency + m_networkLatency));
}
}

// Recreate transport line with new ports value
QString newdata;
Expand Down Expand Up @@ -1243,6 +1280,9 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header,
*m_textStream << "Transport: " << newdata << "\r\n";
*m_textStream << "Session: 1\r\n";
*m_textStream << "Audio-Jack-Status: connected\r\n";

// Ask for master clock value to determine time skew and average network latency
SendTimeRequest();
}
else
{
Expand All @@ -1257,8 +1297,13 @@ void MythRAOPConnection::ProcessRequest(const QStringList &header,
m_nextSequence = RTPseq;
m_nextTimestamp = RTPtimestamp;
}
// Ask for master clock value to determine time skew and average network latency
SendTimeRequest();

// Calculate audio card latency
if (m_cardLatency > 0)
{
*m_textStream << QString("Audio-Latency: %1")
.arg(m_cardLatency+m_networkLatency);
}
}
else if (option == "FLUSH")
{
Expand Down Expand Up @@ -1660,18 +1705,18 @@ int64_t MythRAOPConnection::AudioCardLatency(void)
if (!m_audio)
return 0;

uint64_t timestamp = 123456;

int16_t *samples = (int16_t *)av_mallocz(AVCODEC_MAX_AUDIO_FRAME_SIZE);
int frames = AUDIOCARD_BUFFER * m_frameRate / 1000;
m_audio->AddData((char *)samples,
frames * (m_sampleSize>>3) * m_channels,
timestamp,
0,
frames);
av_free(samples);
usleep(AUDIOCARD_BUFFER * 1000);
uint64_t audiots = m_audio->GetAudiotime();
return (int64_t)timestamp - (int64_t)audiots;
LOG(VB_PLAYBACK, LOG_DEBUG, LOC + QString("AudioCardLatency: ts=%1ms")
.arg(audiots));
return AUDIOCARD_BUFFER - (int64_t)audiots;
}

void MythRAOPConnection::newEventClient(QTcpSocket *client)
Expand Down
2 changes: 1 addition & 1 deletion mythtv/libs/libmythtv/AirPlay/mythraopconnection.h
View file
Expand Up @@ -157,7 +157,7 @@ class MTV_PUBLIC MythRAOPConnection : public QObject
uint64_t m_currentTimestamp;
uint16_t m_nextSequence;
uint64_t m_nextTimestamp;
uint64_t m_bufferLength;
int64_t m_bufferLength;
uint64_t m_timeLastSync;
int64_t m_cardLatency;
int64_t m_adjustedLatency;
Expand Down

0 comments on commit 96f58cb

Please sign in to comment.