AudioServer+Userland: Decouple client sample rates from device rate

This change was a long time in the making ever since we obtained sample
rate awareness in the system. Now, each client has its own sample rate,
accessible via new IPC APIs, and the device sample rate is only
accessible via the management interface. AudioServer takes care of
resampling client streams into the device sample rate. Therefore, the
main improvement introduced with this commit is full responsiveness to
sample rate changes; all open audio programs will continue to play at
correct speed with the audio resampled to the new device rate.

The immediate benefits are manifold:
- Gets rid of the legacy hardware sample rate IPC message in the
  non-managing client
- Removes duplicate resampling and sample index rescaling code
  everywhere
- Avoids potential sample index scaling bugs in SoundPlayer (which have
  happened many times before) and fixes a sample index scaling bug in
  aplay
- Removes several FIXMEs
- Reduces amount of sample copying in all applications (especially
  Piano, where this is critical), improving performance
- Reduces number of resampling users, making future API changes (which
  will need to happen for correct resampling to be implemented) easier

I also threw in a simple race condition fix for Piano's audio player
loop.

Author: kleinesfilmroellchen

Base/usr/share/man/man1/asctl.md

@@ -30,7 +30,7 @@ There are two commands available: `get` reports the state of audio variables, an
 The available variables are:
 * `(v)olume`: Audio server volume, in percent. Integer value.
 * `(m)ute`: Mute state. Boolean value, may be set with `0`, `false` or `1`, `true`.
-* `sample(r)ate`: Sample rate of the sound card. **Attention:** Most audio applications need to be restarted after changing the sample rate. Integer value.
+* `sample(r)ate`: Sample rate of the sound card. Integer value.
 
 Both commands and arguments can be abbreviated: Commands by their first letter, arguments by the letter in parenthesis.
 

Ladybird/AudioCodecPluginLadybird.cpp

@@ -163,7 +163,7 @@ class AudioThread final : public QThread { // We have to use QThread, otherwise
 
                 case AudioTask::Type::Seek:
                     VERIFY(task.data.has_value());
-                    m_position = Web::Platform::AudioCodecPlugin::set_loader_position(m_loader, *task.data, m_duration, audio_output->format().sampleRate());
+                    m_position = Web::Platform::AudioCodecPlugin::set_loader_position(m_loader, *task.data, m_duration);
 
                     if (paused == Paused::Yes)
                         Q_EMIT playback_position_updated(m_position);
@@ -211,10 +211,10 @@ class AudioThread final : public QThread { // We have to use QThread, otherwise
         auto channel_count = audio_output.format().channelCount();
         auto samples_to_load = bytes_available / bytes_per_sample / channel_count;
 
-        auto samples = TRY(Web::Platform::AudioCodecPlugin::read_samples_from_loader(*m_loader, samples_to_load, audio_output.format().sampleRate()));
+        auto samples = TRY(Web::Platform::AudioCodecPlugin::read_samples_from_loader(*m_loader, samples_to_load));
         enqueue_samples(audio_output, io_device, move(samples));
 
-        m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader, audio_output.format().sampleRate());
+        m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader);
         return Paused::No;
     }
 

Userland/Applications/Piano/AudioPlayerLoop.cpp

@@ -17,7 +17,6 @@
 #include <AK/Time.h>
 #include <LibAudio/ConnectionToServer.h>
 #include <LibAudio/Queue.h>
-#include <LibAudio/Resampler.h>
 #include <LibAudio/Sample.h>
 #include <LibIPC/Connection.h>
 #include <LibThreading/Thread.h>
@@ -59,11 +58,7 @@ AudioPlayerLoop::AudioPlayerLoop(TrackManager& track_manager, Atomic<bool>& need
     , m_wav_writer(wav_writer)
 {
     m_audio_client = Audio::ConnectionToServer::try_create().release_value_but_fixme_should_propagate_errors();
-
-    auto target_sample_rate = m_audio_client->get_sample_rate();
-    if (target_sample_rate == 0)
-        target_sample_rate = Music::sample_rate;
-    m_resampler = Audio::ResampleHelper<DSP::Sample>(Music::sample_rate, target_sample_rate);
+    m_audio_client->set_self_sample_rate(sample_rate);
 
     MUST(m_pipeline_thread->set_priority(sched_get_priority_max(0)));
     m_pipeline_thread->start();
@@ -108,15 +103,12 @@ intptr_t AudioPlayerLoop::pipeline_thread_main()
 
 ErrorOr<void> AudioPlayerLoop::send_audio_to_server()
 {
-    TRY(m_resampler->try_resample_into_end(m_remaining_samples, m_buffer));
-
-    auto sample_rate = static_cast<double>(m_resampler->target());
     auto buffer_play_time_ns = 1'000'000'000.0 / (sample_rate / static_cast<double>(Audio::AUDIO_BUFFER_SIZE));
     auto good_sleep_time = Duration::from_nanoseconds(static_cast<unsigned>(buffer_play_time_ns)).to_timespec();
 
     size_t start_of_chunk_to_write = 0;
-    while (start_of_chunk_to_write + Audio::AUDIO_BUFFER_SIZE <= m_remaining_samples.size()) {
-        auto const exact_chunk = m_remaining_samples.span().slice(start_of_chunk_to_write, Audio::AUDIO_BUFFER_SIZE);
+    while (start_of_chunk_to_write + Audio::AUDIO_BUFFER_SIZE <= m_buffer.size()) {
+        auto const exact_chunk = m_buffer.span().slice(start_of_chunk_to_write, Audio::AUDIO_BUFFER_SIZE);
         auto exact_chunk_array = Array<Audio::Sample, Audio::AUDIO_BUFFER_SIZE>::from_span(exact_chunk);
 
         TRY(m_audio_client->blocking_realtime_enqueue(exact_chunk_array, [&]() {
@@ -125,8 +117,8 @@ ErrorOr<void> AudioPlayerLoop::send_audio_to_server()
 
         start_of_chunk_to_write += Audio::AUDIO_BUFFER_SIZE;
     }
-    m_remaining_samples.remove(0, start_of_chunk_to_write);
-    VERIFY(m_remaining_samples.size() < Audio::AUDIO_BUFFER_SIZE);
+    // The buffer has to have been constructed with a size of an integer multiple of the audio buffer size.
+    VERIFY(start_of_chunk_to_write == m_buffer.size());
 
     return {};
 }

Userland/Applications/Piano/AudioPlayerLoop.h

@@ -39,10 +39,8 @@ class AudioPlayerLoop final : public Core::Object {
 
     TrackManager& m_track_manager;
     FixedArray<DSP::Sample> m_buffer;
-    Optional<Audio::ResampleHelper<DSP::Sample>> m_resampler;
     RefPtr<Audio::ConnectionToServer> m_audio_client;
     NonnullRefPtr<Threading::Thread> m_pipeline_thread;
-    Vector<Audio::Sample, Audio::AUDIO_BUFFER_SIZE> m_remaining_samples {};
 
     Atomic<bool> m_should_play_audio { true };
     Atomic<bool> m_exit_requested { false };

Userland/Applications/Piano/Music.h

@@ -8,6 +8,7 @@
 #pragma once
 
 #include <AK/Types.h>
+#include <LibAudio/Queue.h>
 #include <LibGfx/Color.h>
 
 namespace Music {
@@ -23,8 +24,7 @@ struct Sample {
     i16 right;
 };
 
-// HACK: needs to increase with device sample rate, but all of the sample_count stuff is static for now
-constexpr int sample_count = 1 << 10;
+constexpr int sample_count = Audio::AUDIO_BUFFER_SIZE * 10;
 
 constexpr double sample_rate = 44100;
 

Userland/Applications/SoundPlayer/PlaybackManager.cpp

@@ -16,18 +16,16 @@ PlaybackManager::PlaybackManager(NonnullRefPtr<Audio::ConnectionToServer> connec
             return;
         next_buffer();
     }).release_value_but_fixme_should_propagate_errors();
-    m_device_sample_rate = connection->get_sample_rate();
 }
 
 void PlaybackManager::set_loader(NonnullRefPtr<Audio::Loader>&& loader)
 {
     stop();
     m_loader = loader;
     if (m_loader) {
+        m_connection->set_self_sample_rate(loader->sample_rate());
         m_total_length = m_loader->total_samples() / static_cast<float>(m_loader->sample_rate());
-        m_device_samples_per_buffer = PlaybackManager::buffer_size_ms / 1000.0f * m_device_sample_rate;
         m_samples_to_load_per_buffer = PlaybackManager::buffer_size_ms / 1000.0f * m_loader->sample_rate();
-        m_resampler = Audio::ResampleHelper<Audio::Sample>(m_loader->sample_rate(), m_device_sample_rate);
         m_timer->start();
     } else {
         m_timer->stop();
@@ -102,7 +100,7 @@ void PlaybackManager::next_buffer()
     if (m_paused)
         return;
 
-    while (m_connection->remaining_samples() < m_device_samples_per_buffer * always_enqueued_buffer_count) {
+    while (m_connection->remaining_samples() < m_samples_to_load_per_buffer * always_enqueued_buffer_count) {
         bool all_samples_loaded = (m_loader->loaded_samples() >= m_loader->total_samples());
         bool audio_server_done = (m_connection->remaining_samples() == 0);
 
@@ -121,12 +119,6 @@ void PlaybackManager::next_buffer()
         }
         auto buffer = buffer_or_error.release_value();
         m_current_buffer.swap(buffer);
-        VERIFY(m_resampler.has_value());
-
-        m_resampler->reset();
-        // FIXME: Handle OOM better.
-        auto resampled = MUST(FixedArray<Audio::Sample>::create(m_resampler->resample(move(m_current_buffer)).span()));
-        m_current_buffer.swap(resampled);
         MUST(m_connection->async_enqueue(m_current_buffer));
     }
 }

Userland/Applications/SoundPlayer/PlaybackManager.h

@@ -29,7 +29,6 @@ class PlaybackManager final {
     bool toggle_pause();
     void set_loader(NonnullRefPtr<Audio::Loader>&&);
     RefPtr<Audio::Loader> loader() const { return m_loader; }
-    size_t device_sample_rate() const { return m_device_sample_rate; }
 
     bool is_paused() const { return m_paused; }
     float total_length() const { return m_total_length; }
@@ -50,13 +49,10 @@ class PlaybackManager final {
     bool m_paused { true };
     bool m_loop = { false };
     float m_total_length { 0 };
-    size_t m_device_sample_rate { 44100 };
-    size_t m_device_samples_per_buffer { 0 };
     size_t m_samples_to_load_per_buffer { 0 };
     RefPtr<Audio::Loader> m_loader { nullptr };
     NonnullRefPtr<Audio::ConnectionToServer> m_connection;
     FixedArray<Audio::Sample> m_current_buffer;
-    Optional<Audio::ResampleHelper<Audio::Sample>> m_resampler;
     RefPtr<Core::Timer> m_timer;
 
     // Controls the GUI update rate. A smaller value makes the visualizations nicer.

Userland/Applications/SoundPlayer/Player.cpp

@@ -16,13 +16,11 @@ Player::Player(Audio::ConnectionToServer& audio_client_connection)
     m_playback_manager.on_update = [&]() {
         auto samples_played = m_playback_manager.loader()->loaded_samples();
         auto sample_rate = m_playback_manager.loader()->sample_rate();
-        float source_to_dest_ratio = static_cast<float>(sample_rate) / m_playback_manager.device_sample_rate();
-        samples_played *= source_to_dest_ratio;
 
         auto played_seconds = samples_played / sample_rate;
         time_elapsed(played_seconds);
         if (play_state() == PlayState::Playing)
-            sound_buffer_played(m_playback_manager.current_buffer(), m_playback_manager.device_sample_rate(), samples_played);
+            sound_buffer_played(m_playback_manager.current_buffer(), sample_rate, samples_played);
     };
     m_playback_manager.on_finished_playing = [&]() {
         set_play_state(PlayState::Stopped);
@@ -64,8 +62,7 @@ void Player::play_file_path(DeprecatedString const& path)
 
     m_loaded_filename = path;
 
-    // TODO: The combination of sample count, sample rate, and sample data should be properly abstracted for the source and the playback device.
-    total_samples_changed(loader->total_samples() * (static_cast<float>(loader->sample_rate()) / m_playback_manager.device_sample_rate()));
+    total_samples_changed(loader->total_samples());
     m_playback_manager.set_loader(move(loader));
     file_name_changed(path);
 
@@ -156,11 +153,6 @@ void Player::toggle_mute()
 
 void Player::seek(int sample)
 {
-    auto loader = m_playback_manager.loader();
-    if (loader.is_null()) {
-        return;
-    }
-    sample *= (m_playback_manager.device_sample_rate() / static_cast<float>(loader->sample_rate()));
     m_playback_manager.seek(sample);
 }
 

Userland/Libraries/LibAudio/ConnectionToServer.cpp

@@ -16,6 +16,7 @@
 #include <LibAudio/UserSampleQueue.h>
 #include <LibCore/Event.h>
 #include <LibThreading/Mutex.h>
+#include <Userland/Services/AudioServer/AudioClientEndpoint.h>
 #include <sched.h>
 #include <time.h>
 
@@ -37,6 +38,7 @@ ConnectionToServer::ConnectionToServer(NonnullOwnPtr<Core::LocalSocket> socket)
         return (intptr_t) nullptr;
     }))
 {
+    update_good_sleep_time();
     async_pause_playback();
     set_buffer(*m_buffer);
 }
@@ -64,12 +66,12 @@ ErrorOr<void> ConnectionToServer::async_enqueue(FixedArray<Sample>&& samples)
 {
     if (!m_background_audio_enqueuer->is_started()) {
         m_background_audio_enqueuer->start();
+        // Wait until the enqueuer has constructed its loop. A pseudo-spinlock is fine since this happens as soon as the other thread gets scheduled.
         while (!m_enqueuer_loop)
             usleep(1);
         TRY(m_background_audio_enqueuer->set_priority(THREAD_PRIORITY_MAX));
     }
 
-    update_good_sleep_time();
     m_user_queue->append(move(samples));
     // Wake the background thread to make sure it starts enqueuing audio.
     m_enqueuer_loop->post_event(*this, make<Core::CustomEvent>(0));
@@ -86,12 +88,18 @@ void ConnectionToServer::clear_client_buffer()
 
 void ConnectionToServer::update_good_sleep_time()
 {
-    auto sample_rate = static_cast<double>(get_sample_rate());
+    auto sample_rate = static_cast<double>(get_self_sample_rate());
     auto buffer_play_time_ns = 1'000'000'000.0 / (sample_rate / static_cast<double>(AUDIO_BUFFER_SIZE));
     // A factor of 1 should be good for now.
     m_good_sleep_time = Duration::from_nanoseconds(static_cast<unsigned>(buffer_play_time_ns)).to_timespec();
 }
 
+void ConnectionToServer::set_self_sample_rate(u32 sample_rate)
+{
+    IPC::ConnectionToServer<AudioClientEndpoint, AudioServerEndpoint>::set_self_sample_rate(sample_rate);
+    update_good_sleep_time();
+}
+
 // Non-realtime audio writing loop
 void ConnectionToServer::custom_event(Core::CustomEvent&)
 {

Userland/Libraries/LibAudio/ConnectionToServer.h

@@ -55,6 +55,8 @@ class ConnectionToServer final
     // Non-realtime code needn't worry about this.
     size_t remaining_buffers() const;
 
+    void set_self_sample_rate(u32 sample_rate);
+
     virtual void die() override;
 
     Function<void(double volume)> on_client_volume_change;
@@ -67,7 +69,6 @@ class ConnectionToServer final
     // We use this to perform the audio enqueuing on the background thread's event loop
     virtual void custom_event(Core::CustomEvent&) override;
 
-    // FIXME: This should be called every time the sample rate changes, but we just cautiously call it on every non-realtime enqueue.
     void update_good_sleep_time();
 
     // Shared audio buffer: both server and client constantly read and write to/from this.