From 15ef41e5c76a2a0d77c04a710523f5327b1f3639 Mon Sep 17 00:00:00 2001 From: Wu Jianhua Date: Sat, 10 Feb 2024 01:02:38 +0800 Subject: [PATCH] Audio: add CoreAudio support Signed-off-by: Wu Jianhua --- Immortal/Audio/AudioPrecompiledHeader.h | 12 + Immortal/Audio/AudioRenderContext.h | 72 --- Immortal/Audio/AudioSource.cpp | 68 --- Immortal/Audio/AudioSource.h | 86 --- Immortal/Audio/CMakeLists.txt | 47 ++ Immortal/Audio/CoreAudio.h | 124 +++++ Immortal/Audio/CoreAudio.mm | 497 ++++++++++++++++++ Immortal/Audio/Device.cpp | 90 ++-- Immortal/Audio/Device.h | 25 +- ...udioRenderContext.cpp => IAudioDevice.cpp} | 12 +- Immortal/Audio/IAudioDevice.h | 69 +++ Immortal/Audio/WASAPI.cpp | 85 +-- Immortal/Audio/WASAPI.h | 29 +- Immortal/CMakeLists.txt | 25 +- Immortal/Immortal.h | 1 - Immortal/impch.h | 9 - 16 files changed, 877 insertions(+), 374 deletions(-) create mode 100644 Immortal/Audio/AudioPrecompiledHeader.h delete mode 100644 Immortal/Audio/AudioRenderContext.h delete mode 100644 Immortal/Audio/AudioSource.cpp delete mode 100644 Immortal/Audio/AudioSource.h create mode 100644 Immortal/Audio/CMakeLists.txt create mode 100644 Immortal/Audio/CoreAudio.h create mode 100644 Immortal/Audio/CoreAudio.mm rename Immortal/Audio/{AudioRenderContext.cpp => IAudioDevice.cpp} (61%) create mode 100644 Immortal/Audio/IAudioDevice.h diff --git a/Immortal/Audio/AudioPrecompiledHeader.h b/Immortal/Audio/AudioPrecompiledHeader.h new file mode 100644 index 00000000..5a890ce4 --- /dev/null +++ b/Immortal/Audio/AudioPrecompiledHeader.h @@ -0,0 +1,12 @@ +#pragma once + +#include "IAudioDevice.h" +#include "Device.h" + +#ifdef _WIN32 +#include "WASAPI.h" +#elif defined(__APPLE__) +#include "CoreAudio.h" +#elif defined(__linux__) +#include "ALSA.h" +#endif diff --git a/Immortal/Audio/AudioRenderContext.h b/Immortal/Audio/AudioRenderContext.h deleted file mode 100644 index 3ba85dc0..00000000 --- a/Immortal/Audio/AudioRenderContext.h +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright (C) 2022, by Wu Jianhua (toqsxw@outlook.com) - * - * This library is distributed under the Apache-2.0 license. - */ - -#pragma once - -#include "Core.h" - -#define REFTIMES_PER_SEC 10000000ll -#define REFTIMES_PER_MILLISEC 10000ll - -namespace Immortal -{ - -struct WaveFormat -{ - uint32_t Channels; - uint32_t SampleRate; -}; - -class AudioRenderContext -{ -public: - AudioRenderContext() : - bufferFrameCount{}, - format{} - { - format.Channels = 2; - format.SampleRate = 48000; - - bufferFrameCount = format.SampleRate; - } - - virtual ~AudioRenderContext() { } - - virtual void OpenDevice() = 0; - - virtual void Begin() = 0; - - virtual void End() = 0; - - virtual void Reset() = 0; - - virtual void Pause(bool enable) = 0; - - virtual int PlaySamples(uint32_t numberSamples, const uint8_t *pData) = 0; - - virtual double GetPostion() = 0; - -public: - uint32_t GetBufferSize() const - { - return bufferFrameCount; - } - - uint32_t GetSampleRate() const - { - return format.SampleRate; - } - -public: - static AudioRenderContext *CreateInstance(); - -public: - uint32_t bufferFrameCount; - - WaveFormat format; -}; - -} diff --git a/Immortal/Audio/AudioSource.cpp b/Immortal/Audio/AudioSource.cpp deleted file mode 100644 index b0099eca..00000000 --- a/Immortal/Audio/AudioSource.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright (C) 2022, by Wu Jianhua (toqsxw@outlook.com) - * - * This library is distributed under the Apache-2.0 license. - */ - -#include "AudioSource.h" -#include "Helper/Platform.h" - -#ifdef _MSC_VER -#include -#endif - -namespace Immortal -{ - -AudioSource::AudioSource(Ref &codec) : - codec{ codec }, - pts{} -{ - -} - -#ifdef _MSC_VER -HRESULT AudioSource::LoadData(int numFramesAvaiable, void *pData, DWORD *flags) -{ - auto picture = codec->GetPicture(); - float *wave = (float*)&picture.GetData()[pts]; - - auto samples = std::min(picture.GetWidth() * sizeof(float) - pts, numFramesAvaiable * 2 * sizeof(float)); - - if (!samples) - { - *flags |= AUDCLNT_BUFFERFLAGS_SILENT; - return S_OK; - } - memcpy(pData, wave, samples); - pts += samples; - - return S_OK; -} -#endif - -bool AudioSource::InProgress() const -{ - auto picture = codec->GetPicture(); - return pts < picture.GetWidth() * sizeof(float); -} - -AudioClip AudioSource::GetAudioClip(int frames) -{ - auto picture = codec->GetPicture(); - - if (!frames) - { - frames = picture.GetWidth(); - } - - AudioClip clip{ picture }; - clip.pData = (const uint8_t *)&picture.GetData()[pts]; - clip.bytes = frames /* sample rate per frame */ * 2 /* channels */ * sizeof(float) /* format */; - clip.frames = frames; - - pts += clip.bytes; - return clip; -} - -} diff --git a/Immortal/Audio/AudioSource.h b/Immortal/Audio/AudioSource.h deleted file mode 100644 index fbcf4059..00000000 --- a/Immortal/Audio/AudioSource.h +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright (C) 2022, by Wu Jianhua (toqsxw@outlook.com) - * - * This library is distributed under the Apache-2.0 license. - */ - -#pragma once - -#include "Core.h" -#include "Vision/Audio/WAV.h" - -namespace Immortal -{ - -struct AudioClip; -class AudioSource -{ -public: - AudioSource(Ref &codec); - -#ifdef _MSC_VER - HRESULT LoadData(int numFramesAvaiable, void* pData, DWORD* flags); -#endif - - bool InProgress() const; - - AudioClip GetAudioClip(int frames = 0); - -protected: - Ref codec; - - int pts; -}; - -struct AudioClip -{ - AudioClip() : - bytes{}, - pData{}, - frames{}, - picture{} - { - - } - - AudioClip(Picture picture) : - bytes{}, - pData{ picture.GetData() }, - frames{}, - picture{picture} - { - frames = picture.GetWidth(); - bytes = frames << 3; - } - - int Cosume(void *dst, uint32_t frameRequested) - { - size_t samples = frameRequested * 2 * sizeof(float); - memcpy(dst, pData, samples); - pData += samples; - bytes -= samples; - frames -= frameRequested; - - return bytes; - } - - int Read(uint32_t frameRequested) - { - size_t samples = frameRequested * 2 * sizeof(float); - pData += samples; - bytes -= samples; - frames -= frameRequested; - - return bytes; - } - - size_t bytes; - - const uint8_t *pData; - - uint32_t frames; - - Picture picture; -}; - -} diff --git a/Immortal/Audio/CMakeLists.txt b/Immortal/Audio/CMakeLists.txt new file mode 100644 index 00000000..a5766a81 --- /dev/null +++ b/Immortal/Audio/CMakeLists.txt @@ -0,0 +1,47 @@ +cmake_minimum_required(VERSION 3.16) + +set (CMAKE_VERBOSE_MAKEFILE ON) +set_property(GLOBAL PROPERTY USE_FOLDERS ON) + +project("ImmortalAudio" LANGUAGES CXX) + +set(AUDIO_FILES + IAudioDevice.cpp + IAudioDevice.h + Device.cpp + Device.h) + +if (WIN32) + list(APPEND AUDIO_FILES + WASAPI.cpp + WASAPI.h) +elseif(APPLE) + list(APPEND AUDIO_FILES + CoreAudio.mm + CoreAudio.h) +elseif(UNIX) + list(APPEND AUDIO_FILES + ALSA.cpp + ALSA.h) +endif() + +list(TRANSFORM AUDIO_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") + +source_group("\\" FILES ${AUDIO_FILES}) + +add_library(${PROJECT_NAME} STATIC ${AUDIO_FILES}) + +target_link_libraries(${PROJECT_NAME} PUBLIC + ImmortalShared + ImmortalVision) + +if (NOT APPLE) + target_precompile_headers(${PROJECT_NAME} PUBLIC $<$:${CMAKE_CURRENT_SOURCE_DIR}/AudioPrecompiledHeader.h>) +else() + target_link_libraries(${PROJECT_NAME} PRIVATE + "-framework Cocoa" + "-framework IOKit" + "-framework CoreFoundation" + "-framework CoreAudio" + "-framework AudioToolBox") +endif() diff --git a/Immortal/Audio/CoreAudio.h b/Immortal/Audio/CoreAudio.h new file mode 100644 index 00000000..8b60d80c --- /dev/null +++ b/Immortal/Audio/CoreAudio.h @@ -0,0 +1,124 @@ +/** + * Copyright (C) 2024, by Wu Jianhua (toqsxw@outlook.com) + * + * This library is distributed under the Apache-2.0 license. + */ + +#pragma once + + +#ifndef COREAUDIO_CONTEXT_H_ +#define COREAUDIO_CONTEXT_H_ + +#include "Core.h" +#include "Shared/IObject.h" +#include "Shared/Async.h" +#include "IAudioDevice.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Immortal +{ +namespace CoreAudio +{ + +struct AudioBuffer +{ + uint8_t *data; + size_t size; +}; + +class Device : public IAudioDevice +{ +public: + using Super = IAudioDevice; + SL_SWAPPABLE(Device) + +public: + Device(); + + virtual ~Device() override; + + virtual void OpenDevice() override; + + virtual void Begin() override; + + virtual void End() override; + + virtual void Reset() override; + + virtual void Pause(bool enable) override; + + virtual double GetPostion() override; + + virtual void BeginRender(uint32_t frames) override; + + virtual void EndRender(uint32_t frames) override; + + virtual void WriteBuffer(const uint8_t *buffer, size_t size) override; + + virtual uint32_t GetAvailableFrameCount() override; + + virtual AudioFormat GetFormat() override; + + void RefreshPhysicalDevices(); + + void CheckDevice(std::vector &devices, AudioDeviceType deviceType); + + void ConnectDeviceToAudioQueue(AudioDeviceType type); + + void OnBufferReady(AudioQueueRef audioQueue, AudioQueueBufferRef buffer); + + void Release(); + +public: + void Swap(Device &other) + { + std::swap_ranges(&handle, &handle + SL_ARRAY_LENGTH(handle), &other.handle ); + std::swap_ranges(&channels, &channels + SL_ARRAY_LENGTH(channels), &other.channels); + + std::swap(queue, other.queue ); + std::swap(buffers, other.buffers ); + std::swap(description, other.description ); + std::swap(availableFrames, other.availableFrames); + std::swap(framesPerBuffer, other.framesPerBuffer); + std::swap(memoryResource, other.memoryResource ); + std::swap(bufferQueue, other.bufferQueue ); + std::swap(data, other.data ); + std::swap(mutex, other.mutex ); + } + +protected: + AudioDeviceID handle[2]; + + AudioQueueRef queue; + + std::vector buffers; + + AudioStreamBasicDescription description; + + uint32_t availableFrames; + + uint32_t framesPerBuffer; + + std::vector memoryResource; + + std::queue bufferQueue; + + uint8_t *data; + + URef mutex; + + uint8_t channels[2]; +}; + +} +} + +#endif diff --git a/Immortal/Audio/CoreAudio.mm b/Immortal/Audio/CoreAudio.mm new file mode 100644 index 00000000..2ea98286 --- /dev/null +++ b/Immortal/Audio/CoreAudio.mm @@ -0,0 +1,497 @@ +/** + * Copyright (C) 2024, by Wu Jianhua (toqsxw@outlook.com) + * + * This library is distributed under the Apache-2.0 license. + */ + +#include "CoreAudio.h" +#include +#include +#include +#include +#include "Audio/IAudioDevice.h" +#include "Shared/Log.h" +#include "Shared/Async.h" +#include "Graphics/Format.h" + +namespace Immortal +{ + +namespace CoreAudio +{ + +static const AudioObjectPropertyAddress DevicesAddress = { + .mSelector = kAudioHardwarePropertyDevices, + .mScope = kAudioObjectPropertyScopeGlobal, + .mElement = kAudioObjectPropertyElementMain +}; + +static AudioChannelLayoutTag GetAudioChannelLayout(int channels) +{ + switch (channels) + { + case 1: + return kAudioChannelLayoutTag_Mono; + break; + case 2: + return kAudioChannelLayoutTag_Stereo; + + case 3: + return kAudioChannelLayoutTag_DVD_4; + + case 4: + return kAudioChannelLayoutTag_Quadraphonic; + + case 5: + return kAudioChannelLayoutTag_MPEG_5_0_A; + + case 6: + return kAudioChannelLayoutTag_MPEG_5_1_A; + + case 7: + return kAudioChannelLayoutTag_MPEG_6_1_A; + + case 8: + return kAudioChannelLayoutTag_MPEG_7_1_A; + + default: + return kAudioChannelLayoutTag_Unknown; + } +} + +static void OnBufferReadyCallback(void *data, AudioQueueRef audioQueue, AudioQueueBufferRef buffer) +{ + Device *This = (Device *)data; + This->OnBufferReady(audioQueue, buffer); +} + +static OSStatus DeviceListChangedNotification(AudioObjectID systemObj, UInt32 num_addr, const AudioObjectPropertyAddress *addrs, void *data) +{ + Device *This = (Device *)data; + This->RefreshPhysicalDevices(); + return kAudioHardwareNoError; +} + +Device::Device() : + Super{}, + handle{}, + queue{}, + buffers{}, + description{}, + availableFrames{}, + framesPerBuffer{}, + memoryResource{}, + bufferQueue{}, + data{}, + mutex{ new std::mutex{} }, + channels{} +{ + OpenDevice(); +} + +Device::~Device() +{ + Release(); +} + +void Device::OpenDevice() +{ + Release(); + RefreshPhysicalDevices(); + + AudioObjectPropertyAddress defaultOutputDeviceAddress = { + .mSelector = kAudioHardwarePropertyDefaultOutputDevice, + .mScope = kAudioObjectPropertyScopeGlobal, + .mElement = kAudioObjectPropertyElementMain + }; + + AudioObjectAddPropertyListener(kAudioObjectSystemObject, &DevicesAddress, DeviceListChangedNotification, (void *)this); + + AudioDeviceID device = handle[0]; + + AudioDeviceID defaultDevice = 0; + uint32_t size = sizeof(AudioDeviceID); + OSStatus status = AudioObjectGetPropertyData(kAudioObjectSystemObject, &defaultOutputDeviceAddress, 0, nullptr, &size, &defaultDevice); + if (status != kAudioHardwareNoError) + { + LOG::ERR("CoreAudio: failed to query the default output device!"); + return; + } + + description = AudioStreamBasicDescription { + .mSampleRate = description.mSampleRate, + .mFormatID = kAudioFormatLinearPCM, + .mFormatFlags = kLinearPCMFormatFlagIsPacked | kLinearPCMFormatFlagIsFloat, + .mFramesPerPacket = 1, + .mChannelsPerFrame = channels[0], + }; + + description.mBitsPerChannel = sizeof(float) * 8; + description.mBytesPerFrame = description.mChannelsPerFrame * description.mBitsPerChannel / 8; + description.mBytesPerPacket = description.mBytesPerFrame * description.mFramesPerPacket; + + AudioObjectPropertyAddress address = { + .mSelector = kAudioDevicePropertyDeviceIsAlive, + .mScope = kAudioDevicePropertyScopeOutput, + .mElement = kAudioObjectPropertyElementMain + }; + + uint32_t alive = 0; + size = sizeof(alive); + status = AudioObjectGetPropertyData(device, &address, 0, nullptr, &size, &alive); + + if (status == kAudioHardwareNoError && !alive) + { + LOG::ERR("CoreAudio: requested device existed but not alive!"); + return; + } + + pid_t pid = 0; + size = sizeof(pid); + address.mSelector = kAudioDevicePropertyHogMode; + status = AudioObjectGetPropertyData(device, &address, 0, nullptr, &size, &pid); + if (status == kAudioHardwareNoError && pid != -1) + { + LOG::ERR("CoreAudio: requested device is hogged"); + } + + status = AudioQueueNewOutput(&description, OnBufferReadyCallback, (void *)this, CFRunLoopGetCurrent(), kCFRunLoopCommonModes, 0, &queue); + if (status != kAudioHardwareNoError) + { + LOG::ERR("CoreAudio::AudioQueueNewOutput: failed to allocate output!"); + return; + } + + ConnectDeviceToAudioQueue(AudioDeviceType::Ouput); + + AudioChannelLayout layout = {}; + layout.mChannelLayoutTag = GetAudioChannelLayout(description.mChannelsPerFrame); + if (layout.mChannelLayoutTag != kAudioChannelLayoutTag_Unknown) + { + status = AudioQueueSetProperty(queue, kAudioQueueProperty_ChannelLayout, &layout, sizeof(layout)); + if (status != kAudioHardwareNoError) + { + LOG::ERR("CoreAudio::AudioQueue: failed to set property - channel layout!"); + } + } + + Format format = Format::VECTOR2; + framesPerBuffer = 1024; + size_t bufferSize = framesPerBuffer * format.GetTexelSize(); + + size_t audioBufferCount = 2; + buffers.resize(audioBufferCount); + for (auto &buffer : buffers) + { + status = AudioQueueAllocateBuffer(queue, (uint32_t)bufferSize, &buffer); + if (status != kAudioHardwareNoError) + { + LOG::ERR("CoreAudio::AudioQueue: failed to allocate buffer!"); + return; + } + memset(buffer->mAudioData, 0, buffer->mAudioDataBytesCapacity); + buffer->mAudioDataByteSize = buffer->mAudioDataBytesCapacity; + status = AudioQueueEnqueueBuffer(queue, buffer, 0, nullptr); + if (status != kAudioHardwareNoError) + { + LOG::ERR("CoreAudio::AudioQueue: failed to enqueue buffer!"); + return; + } + } + + status = AudioQueueStart(queue, nullptr); + if (status != kAudioHardwareNoError) + { + LOG::ERR("CoreAudio: faild to start audio queue!"); + } + + memoryResource.resize(format.GetTexelSize() * description.mSampleRate * 2); +} + +void Device::Begin() +{ + AudioQueueStart(queue, 0); +} + +void Device::End() +{ + AudioQueueStop(queue, 0); +} + +void Device::Reset() +{ + { + std::lock_guard lock{ *mutex }; + bufferQueue = {}; + availableFrames = 0; + } + AudioQueueFlush(queue); +} + +void Device::Pause(bool enabled) +{ + if (enabled) + { + AudioQueuePause(queue); + } + else + { + AudioQueueStart(queue, nullptr); + } +} + +double Device::GetPostion() +{ + return 0.0f; +} + +void Device::BeginRender(uint32_t frames) +{ + mutex->lock(); + data = &memoryResource.back() - availableFrames * description.mBytesPerFrame; + availableFrames -= frames; +} + +void Device::EndRender(uint32_t frames) +{ + bufferQueue.push(AudioBuffer{ (uint8_t *)data, frames * description.mBytesPerFrame }); + data = nullptr; + mutex->unlock(); +} + +void Device::WriteBuffer(const uint8_t *buffer, size_t size) +{ + memcpy(data, buffer, size); +} + +uint32_t Device::GetAvailableFrameCount() +{ + if (availableFrames == 0) + { + availableFrames = description.mSampleRate * 2; + } + + return availableFrames; +} + +AudioFormat Device::GetFormat() +{ + AudioFormat ret = { + .format = Format::VECTOR2, + .channels = description.mChannelsPerFrame, + .silence = 0, + .sampleRate = description.mSampleRate, + }; + + return ret; +} + +void Device::RefreshPhysicalDevices() +{ + uint32_t size = 0; + + OSStatus status = AudioObjectGetPropertyDataSize(kAudioObjectSystemObject, &DevicesAddress, 0, nullptr, &size); + if (status != kAudioHardwareNoError) + { + return; + } + + uint32_t deviceCount = size / sizeof(AudioDeviceID); + std::vector devices; + devices.resize(deviceCount); + status = AudioObjectGetPropertyData(kAudioObjectSystemObject, &DevicesAddress, 0, nullptr, &size, devices.data()); + if (status != kAudioHardwareNoError) + { + return; + } + + CheckDevice(devices, AudioDeviceType::Ouput); + CheckDevice(devices, AudioDeviceType::Capture); +} + +void Device::CheckDevice(std::vector &devices, AudioDeviceType deviceType) +{ + AudioObjectPropertyScope scope = deviceType == AudioDeviceType::Capture ? kAudioDevicePropertyScopeInput : kAudioDevicePropertyScopeOutput; + AudioObjectPropertyElement element = kAudioObjectPropertyElementMain; + + AudioObjectPropertyAddress address = { + .mSelector = kAudioDevicePropertyStreamConfiguration, + .mScope = scope, + .mElement = element, + }; + + AudioObjectPropertyAddress nameAddress = { + .mSelector = kAudioObjectPropertyName, + .mScope = scope, + .mElement = element, + }; + + AudioObjectPropertyAddress sampleRateAddress = { + .mSelector = kAudioDevicePropertyNominalSampleRate, + .mScope = scope, + .mElement = element, + }; + + std::vector audioBufferLists; + for (size_t i = 0; i < devices.size(); i++) + { + auto &device = devices[i]; + if (device == 0) + { + continue; + } + + uint32_t size = 0; + OSStatus status = AudioObjectGetPropertyDataSize(device, &address, 0, nullptr, &size); + if (status != kAudioHardwareNoError) + { + continue;; + } + + std::unique_ptr audioBufferList; + audioBufferList.reset((AudioBufferList *)new uint8_t[size]); + + status = AudioObjectGetPropertyData(device, &address, 0, nullptr, &size, audioBufferList.get()); + if (status != kAudioHardwareNoError) + { + return; + } + + auto &channel = channels[(int)deviceType]; + for (int j = 0; j < audioBufferList->mNumberBuffers; j++) + { + channel += audioBufferList->mBuffers[j].mNumberChannels; + } + audioBufferList.reset(); + + if (channel == 0) + { + continue; + } + + size = sizeof(description.mSampleRate); + status = AudioObjectGetPropertyData(device, &sampleRateAddress, 0, nullptr, &size, &description.mSampleRate); + if (status != kAudioHardwareNoError) + { + LOG::ERR("CoreAudio: failed to query sample rate!"); + } + + CFStringRef stringRef = nullptr; + size = sizeof(CFStringRef); + status = AudioObjectGetPropertyData(device, &nameAddress, 0, nullptr, &size, &stringRef); + if (status != kAudioHardwareNoError) + { + LOG::ERR("CoreAudio: failed to query device name!"); + continue; + } + + CFIndex length = CFStringGetMaximumSizeForEncoding(CFStringGetLength(stringRef), kCFStringEncodingUTF8); + std::string name; + name.resize(length); + if (CFStringGetCString(stringRef, name.data(), name.size(), kCFStringEncodingUTF8)) + { + while (length > 0 && name[length - 1] == ' ') + { + length--; + } + if (length < name.size()) + { + name[length] = '\0'; + } + + LOG::INFO("CoreAudio: found {} device[{}] - {} - {}", deviceType == AudioDeviceType::Capture ? "capture" : "output", i, name, (int)device); + } + CFRelease(stringRef); + + handle[(int)deviceType] = device; + device = 0; + } +} + +void Device::ConnectDeviceToAudioQueue(AudioDeviceType type) +{ + AudioDeviceID device = handle[(int)type]; + const AudioObjectPropertyAddress deviceUIDAddress = { + kAudioDevicePropertyDeviceUID, + type == AudioDeviceType::Capture ? kAudioDevicePropertyScopeInput : kAudioDevicePropertyScopeOutput, + kAudioObjectPropertyElementMain + }; + + CFStringRef deviceUID = {}; + uint32_t size = sizeof(CFStringRef); + if (AudioObjectGetPropertyData(device, &deviceUIDAddress, 0, nullptr, &size, &deviceUID) != kAudioHardwareNoError || + AudioQueueSetProperty(queue, kAudioQueueProperty_CurrentDevice, &deviceUID, size) != kAudioHardwareNoError) + + { + LOG::ERR("CoreAudio: failed to connect device to the audio queue!"); + return; + } + + return; +} + +void Device::OnBufferReady(AudioQueueRef audioQueue, AudioQueueBufferRef buffer) +{ + buffer->mAudioDataByteSize = 0; + while (true) + { + mutex->lock(); + if (bufferQueue.empty()) + { + mutex->unlock(); + break; + } + auto &b = bufferQueue.front(); + mutex->unlock(); + + uint8_t *data = (uint8_t *)buffer->mAudioData; + data += buffer->mAudioDataByteSize; + + auto copySize = std::min(b.size, (size_t)buffer->mAudioDataBytesCapacity - buffer->mAudioDataByteSize); + memcpy(data, b.data, copySize); + buffer->mAudioDataByteSize += copySize; + b.size -= copySize; + b.data += copySize; + + if (b.size == 0) + { + mutex->lock(); + bufferQueue.pop(); + mutex->unlock(); + } + if (buffer->mAudioDataByteSize == buffer->mAudioDataBytesCapacity) + { + break; + } + } + + if (buffer->mAudioDataByteSize == 0) + { + buffer->mAudioDataByteSize = buffer->mAudioDataBytesCapacity; + memset(buffer->mAudioData, 0, buffer->mAudioDataBytesCapacity); + } + AudioQueueEnqueueBuffer(queue, buffer, 0, nullptr); +} + +void Device::Release() +{ + if (queue) + { + for (auto &buffer : buffers) + { + AudioQueueFreeBuffer(queue, buffer); + } + buffers.clear(); + + AudioQueueFlush(queue); + AudioQueueStop(queue, 0); + AudioQueueDispose(queue, 0); + } + + std::lock_guard lock{ *mutex }; + bufferQueue = {}; + memoryResource = {}; +} + +} +} diff --git a/Immortal/Audio/Device.cpp b/Immortal/Audio/Device.cpp index c35f464d..7d4428fb 100644 --- a/Immortal/Audio/Device.cpp +++ b/Immortal/Audio/Device.cpp @@ -5,7 +5,6 @@ */ #include "Device.h" -#include "AudioSource.h" namespace Immortal { @@ -22,7 +21,7 @@ struct StereoVector2 }; AudioDevice::AudioDevice() : - context{ AudioRenderContext::CreateInstance() }, + handle{ IAudioDevice::CreateInstance() }, pts{ 0 }, samples{ 0 }, stopping{ false }, @@ -31,17 +30,19 @@ AudioDevice::AudioDevice() : status{false} { instance = this; - if (!context) + if (!handle) { return; } thread = new Thread{ [=, this] { uint64_t duration = 0; - context->Begin(); + handle->Begin(); StereoVector2 buffer[2048] = {}; StereoVector2 *ptr = buffer; + AudioFormat format = handle->GetFormat(); + static int lastSamples = 1024; while (!stopping) { @@ -66,10 +67,10 @@ AudioDevice::AudioDevice() : size_t bytes = picture.GetWidth() << 3; memcpy(ptr, picture.GetData(), bytes); ptr += picture.GetWidth(); - size_t frames = ptr - buffer; + uint32_t frames = uint32_t(ptr - buffer); if (frames > 1024) { - frameLeft = context->PlaySamples(frames, (const uint8_t *)buffer); + frameLeft = PlaySamples(frames, (const uint8_t *)buffer); ptr = buffer; } } @@ -77,15 +78,15 @@ AudioDevice::AudioDevice() : { pts = picture.GetTimestamp(); samples = picture.GetWidth(); - frameLeft = context->PlaySamples(samples, picture.GetData()); + frameLeft = PlaySamples(samples, picture.GetData()); } - duration = Seconds2Nanoseconds(((float)frameLeft / context->GetSampleRate())); + duration = Seconds2Nanoseconds(((float) frameLeft / format.sampleRate)); } else { if (lastSamples >= 512) { - duration = Seconds2Nanoseconds(((float)1024 / context->GetSampleRate())); + duration = Seconds2Nanoseconds(((float)1024 / format.sampleRate)); } } @@ -95,7 +96,7 @@ AudioDevice::AudioDevice() : status.wait(true); } - context->End(); + handle->End(); } }; thread->Start(); @@ -115,41 +116,15 @@ AudioDevice::~AudioDevice() } } -void AudioDevice::PlayAudioStream(AudioSource *pAudioSource) -{ - uint64_t duration = Seconds2Nanoseconds((float)context->GetBufferSize() / context->GetSampleRate()); - - context->Begin(); - { - auto audioClip = pAudioSource->GetAudioClip(); - context->PlaySamples(audioClip.frames, audioClip.pData); - - uint64_t duration = Seconds2Nanoseconds(((float)audioClip.frames / context->GetSampleRate())); - std::this_thread::sleep_for(std::chrono::nanoseconds(duration >> 1)); - } - context->End(); - std::this_thread::sleep_for(std::chrono::nanoseconds(duration >> 1)); -} - -void AudioDevice::PlayClip(AudioClip pAudioClip) -{ - -} - -void AudioDevice::PlayFrame(Picture picture) -{ - -} - void AudioDevice::OnPauseDown() { status = true; - context->Pause(true); + handle->Pause(true); } void AudioDevice::OnPauseRelease() { - context->Pause(false); + handle->Pause(false); status = false; status.notify_one(); } @@ -157,35 +132,42 @@ void AudioDevice::OnPauseRelease() void AudioDevice::Reset() { reset = true; - context->Reset(); + handle->Reset(); } double AudioDevice::GetPosition() const { - return context->GetPostion(); + return handle->GetPostion(); } -double AudioDevice::Sync(uint64_t videoTimestamp, double framesPerSecond, double delta) +int AudioDevice::GetSampleRate() const { - double start = (double)startpts * ((double)samples / context->GetSampleRate()); - double time = (double)videoTimestamp * (1.0 / framesPerSecond) + delta; - return time - (start + GetPosition()); + AudioFormat format = handle->GetFormat(); + return format.sampleRate; } -uint64_t AudioDevice::Sync(double framesPerSecond) +int AudioDevice::PlaySamples(uint32_t numberSamples, const uint8_t *pSamples) { - double audioTimestamp = (double)startpts * 512.0 / 48000.0; // ((double)samples / context->GetSampleRate()); - audioTimestamp += GetPosition(); + uint32_t frameRequested = 0; + while (numberSamples > 0) + { + uint32_t numFramesPadding = handle->GetAvailableFrameCount(); - double videoTimestamp = audioTimestamp / (1.0 / framesPerSecond); - return videoTimestamp; -} + frameRequested = std::min(numberSamples, numFramesPadding); -AudioDevice *AudioDevice::instance; + handle->BeginRender(frameRequested); -int AudioDevice::GetSampleRate() -{ - return instance ? instance->context->GetSampleRate() : 48000; + uint32_t bytes = frameRequested << 3; + handle->WriteBuffer(pSamples, bytes); + pSamples += bytes; + numberSamples -= frameRequested; + + handle->EndRender(frameRequested); + } + + return frameRequested; } +AudioDevice *AudioDevice::instance; + } diff --git a/Immortal/Audio/Device.h b/Immortal/Audio/Device.h index be8c0636..051cd7e0 100644 --- a/Immortal/Audio/Device.h +++ b/Immortal/Audio/Device.h @@ -9,27 +9,19 @@ #include "Core.h" #include "Shared/Async.h" #include "Shared/IObject.h" -#include "Audio/AudioSource.h" -#include "AudioRenderContext.h" +#include "IAudioDevice.h" +#include "Vision/Picture.h" namespace Immortal { -class AudioClip; -class AudioSource; -class AudioDevice : public IObject +class IMMORTAL_API AudioDevice : public IObject { public: AudioDevice(); ~AudioDevice(); - void PlayAudioStream(AudioSource *pAudioSource); - - void PlayClip(AudioClip pAudioClip); - - void PlayFrame(Picture picture); - void Reset(); void OnPauseDown(); @@ -37,10 +29,10 @@ class AudioDevice : public IObject void OnPauseRelease(); double GetPosition() const; + + int GetSampleRate() const; - double Sync(uint64_t videoTimestamp, double framesPerSecond, double delta); - - uint64_t Sync(double framesPerSecond); + int PlaySamples(uint32_t numberSamples, const uint8_t *pSamples); public: template @@ -54,16 +46,13 @@ class AudioDevice : public IObject callBack = {}; } -public: - static int GetSampleRate(); - protected: static AudioDevice *instance; protected: URef thread; - URef context; + URef handle; std::mutex mutex; diff --git a/Immortal/Audio/AudioRenderContext.cpp b/Immortal/Audio/IAudioDevice.cpp similarity index 61% rename from Immortal/Audio/AudioRenderContext.cpp rename to Immortal/Audio/IAudioDevice.cpp index 6234b06b..dcb23257 100644 --- a/Immortal/Audio/AudioRenderContext.cpp +++ b/Immortal/Audio/IAudioDevice.cpp @@ -4,27 +4,33 @@ * This library is distributed under the Apache-2.0 license. */ -#include "AudioRenderContext.h" +#include "IAudioDevice.h" #ifdef _WIN32 #include "WASAPI.h" #elif defined(__linux__) #include "ALSA.h" +#elif defined(__APPLE__) +#include "CoreAudio.h" #endif namespace Immortal { -AudioRenderContext *AudioRenderContext::CreateInstance() +IAudioDevice *IAudioDevice::CreateInstance() { #ifdef WASAPI_CONTEXT_H_ - return new WASAPIContext; + return new WASAPI::Device; #endif #ifdef ALSA_CONTEXT_H_ return new ALSAContext; #endif +#ifdef COREAUDIO_CONTEXT_H_ + return new CoreAudio::Device; +#endif + return nullptr; } diff --git a/Immortal/Audio/IAudioDevice.h b/Immortal/Audio/IAudioDevice.h new file mode 100644 index 00000000..bb017b15 --- /dev/null +++ b/Immortal/Audio/IAudioDevice.h @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2022, by Wu Jianhua (toqsxw@outlook.com) + * + * This library is distributed under the Apache-2.0 license. + */ + +#pragma once + +#include "Core.h" +#include "Graphics/Format.h" + +#define REFTIMES_PER_SEC 10000000ll +#define REFTIMES_PER_MILLISEC 10000ll + +namespace Immortal +{ + +enum class AudioDeviceType +{ + Ouput, + Capture +}; + +struct AudioFormat +{ + Format format; + uint8_t channels; + uint8_t silence; + uint32_t sampleRate; +}; + +struct AudioBuffer +{ + uint8_t *data; + uint32_t size; +}; + +class IAudioDevice +{ +public: + virtual ~IAudioDevice() = default; + + virtual void OpenDevice() = 0; + + virtual void Begin() = 0; + + virtual void End() = 0; + + virtual void Reset() = 0; + + virtual void Pause(bool enable) = 0; + + virtual double GetPostion() = 0; + + virtual void BeginRender(uint32_t frames) = 0; + + virtual void WriteBuffer(const uint8_t *buffer, size_t size) = 0; + + virtual void EndRender(uint32_t frames) = 0; + + virtual uint32_t GetAvailableFrameCount() = 0; + + virtual AudioFormat GetFormat() = 0; + +public: + static IAudioDevice *CreateInstance(); +}; + +} diff --git a/Immortal/Audio/WASAPI.cpp b/Immortal/Audio/WASAPI.cpp index 2f800848..6e8235fa 100644 --- a/Immortal/Audio/WASAPI.cpp +++ b/Immortal/Audio/WASAPI.cpp @@ -8,6 +8,8 @@ namespace Immortal { +namespace WASAPI +{ static inline void Check(HRESULT hr) { @@ -17,19 +19,20 @@ static inline void Check(HRESULT hr) } } -WASAPIContext::WASAPIContext() : +Device::Device() : Super{}, - waveFormat{} + waveFormat{}, + bufferSize{} { OpenDevice(); } -WASAPIContext::~WASAPIContext() +Device::~Device() { Release(); } -void WASAPIContext::OpenDevice() +void Device::OpenDevice() { Release(); @@ -49,28 +52,25 @@ void WASAPIContext::OpenDevice() Check(audioClient->GetService(IID_PPV_ARGS(&clock))); - Check(audioClient->GetBufferSize(&bufferFrameCount)); - - format.Channels = waveFormat->nChannels; - format.SampleRate = waveFormat->nSamplesPerSec; + Check(audioClient->GetBufferSize(&bufferSize)); } -void WASAPIContext::Begin() +void Device::Begin() { Check(audioClient->Start()); } -void WASAPIContext::End() +void Device::End() { Check(audioClient->Stop()); } -void WASAPIContext::Reset() +void Device::Reset() { Check(audioClient->Reset()); } -void WASAPIContext::Pause(bool enable) +void Device::Pause(bool enable) { if (enable) { @@ -82,43 +82,55 @@ void WASAPIContext::Pause(bool enable) } } -int WASAPIContext::PlaySamples(uint32_t numberSamples, const uint8_t *pSamples) +double Device::GetPostion() { - uint32_t frameRequested = 0; - while (numberSamples > 0) - { - uint8_t *pData; - - uint32_t numFramesPadding; - Check(audioClient->GetCurrentPadding(&numFramesPadding)); + uint64_t position; + Check(clock->GetPosition(&position, nullptr)); - frameRequested = std::min(numberSamples, bufferFrameCount - numFramesPadding); + uint64_t frequency; + Check(clock->GetFrequency(&frequency)); - Check(renderClient->GetBuffer(frameRequested, &pData)); + return (double)position / (double)frequency; +} - uint32_t bytes = frameRequested << 3; - memcpy(pData, pSamples, bytes); - pSamples += bytes; - numberSamples -= frameRequested; +void Device::BeginRender(uint32_t frames) +{ + Check(renderClient->GetBuffer(frames, &data)); +} - Check(renderClient->ReleaseBuffer(frameRequested, 0)); - } +void Device::WriteBuffer(const uint8_t *buffer, size_t size) +{ + SLASSERT(data != nullptr && "BeginRender is not called yet!"); + memcpy(data, buffer, size); +} - return frameRequested; +void Device::EndRender(uint32_t frames) +{ + Check(renderClient->ReleaseBuffer(frames, 0)); + data = nullptr; } -double WASAPIContext::GetPostion() +uint32_t Device::GetAvailableFrameCount() { - uint64_t position; - Check(clock->GetPosition(&position, nullptr)); + uint32_t padding = 0; + Check(audioClient->GetCurrentPadding(&padding)); - uint64_t frequency; - Check(clock->GetFrequency(&frequency)); + return bufferSize - padding; +} - return (double)position / (double)frequency; +AudioFormat Device::GetFormat() +{ + AudioFormat format = { + .format = Format::VECTOR2, + .channels = (uint8_t)waveFormat->nChannels, + .silence = 0, + .sampleRate = waveFormat->nSamplesPerSec, + }; + + return format; } -void WASAPIContext::Release() +void Device::Release() { if (waveFormat) { @@ -128,3 +140,4 @@ void WASAPIContext::Release() } } +} diff --git a/Immortal/Audio/WASAPI.h b/Immortal/Audio/WASAPI.h index 785530e3..29759f81 100644 --- a/Immortal/Audio/WASAPI.h +++ b/Immortal/Audio/WASAPI.h @@ -9,7 +9,7 @@ #ifndef WASAPI_CONTEXT_H_ #define WASAPI_CONTEXT_H_ -#include "AudioRenderContext.h" +#include "IAudioDevice.h" #include #include @@ -18,17 +18,19 @@ namespace Immortal { +namespace WASAPI +{ using Microsoft::WRL::ComPtr; -class WASAPIContext : public AudioRenderContext +class Device : public IAudioDevice { public: - using Super = AudioRenderContext; + using Super = IAudioDevice; public: - WASAPIContext(); + Device(); - virtual ~WASAPIContext(); + virtual ~Device(); virtual void OpenDevice() override; @@ -40,10 +42,18 @@ class WASAPIContext : public AudioRenderContext virtual void Pause(bool enable) override; - virtual int PlaySamples(uint32_t numberSamples, const uint8_t *pData) override; - virtual double GetPostion() override; + virtual void BeginRender(uint32_t frames) override; + + virtual void WriteBuffer(const uint8_t *buffer, size_t size) override; + + virtual void EndRender(uint32_t frames) override; + + virtual uint32_t GetAvailableFrameCount() override; + + virtual AudioFormat GetFormat() override; + void Release(); protected: @@ -57,11 +67,16 @@ class WASAPIContext : public AudioRenderContext ComPtr clock; + uint32_t bufferSize; + std::mutex mutex; WAVEFORMATEX *waveFormat; + + uint8_t *data; }; +} } #endif diff --git a/Immortal/CMakeLists.txt b/Immortal/CMakeLists.txt index 6f3b76dc..85448320 100644 --- a/Immortal/CMakeLists.txt +++ b/Immortal/CMakeLists.txt @@ -16,14 +16,6 @@ set(ALGORITHM_FILES Algorithm/LightVector.h Algorithm/Rotate.h) -set(AUDIO_FILES - AudioRenderContext.cpp - AudioRenderContext.h - AudioSource.cpp - AudioSource.h - Device.cpp - Device.h) - set(FRAMEWORK_FILES Framework/Application.cpp Framework/Application.h @@ -144,10 +136,6 @@ if (WIN32) list(APPEND NET_FILES Win32Socket.cpp) - list(APPEND AUDIO_FILES - WASAPI.cpp - WASAPI.h) - list(APPEND HELPER_FILES Win32Platform.cpp) @@ -160,10 +148,6 @@ elseif(UNIX) list(APPEND NET_FILES UnixSocket.cpp) - list(APPEND AUDIO_FILES - ALSA.cpp - ALSA.h) - list(APPEND HELPER_FILES LinuxPlatform.cpp) @@ -171,14 +155,12 @@ elseif(UNIX) UnixSemaphore.cpp) endif() -list(TRANSFORM AUDIO_FILES PREPEND "Audio/" ) list(TRANSFORM HELPER_FILES PREPEND "Helper/" ) list(TRANSFORM NET_FILES PREPEND "Net/" ) list(TRANSFORM SYNC_FILES PREPEND "Sync/" ) source_group("\\" FILES ${CORE_FILES} ) source_group("Algorithm\\" FILES ${ALGORITHM_FILES} ) -source_group("Audio\\" FILES ${AUDIO_FILES} ) source_group("Framework\\" FILES ${FRAMEWORK_FILES} ) source_group("Editor\\" FILES ${EDITOR_FILES} ) source_group("Image\\" FILES ${IMAGE_FILES} ) @@ -200,7 +182,6 @@ source_group("Net\\" FILES ${NET_FILES} ) set(PROJECT_FILES ${CORE_FILES} ${ALGORITHM_FILES} - ${AUDIO_FILES} ${FRAMEWORK_FILES} ${EDITOR_FILES} ${FS_FILES} @@ -227,6 +208,7 @@ source_group("\\" FILES ${PROJECT_FILES}) list(TRANSFORM PROJECT_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") add_subdirectory(Shared) +add_subdirectory(Audio) add_subdirectory(Graphics) add_subdirectory(Vision) @@ -242,7 +224,9 @@ else() target_link_libraries(${PROJECT_NAME} PRIVATE "-framework Cocoa" "-framework IOKit" - "-framework CoreFoundation") + "-framework CoreFoundation" + "-framework CoreAudio" + "-framework AudioToolBox") endif() add_subdirectory(slapi) @@ -251,5 +235,6 @@ target_link_libraries(${PROJECT_NAME} PUBLIC slapi external imgui + ImmortalAudio ImmortalGraphics ImmortalVision) diff --git a/Immortal/Immortal.h b/Immortal/Immortal.h index b70584d2..71f58526 100644 --- a/Immortal/Immortal.h +++ b/Immortal/Immortal.h @@ -6,7 +6,6 @@ #include #include "Audio/Device.h" -#include "Audio/AudioSource.h" #include "Core.h" #include "Config.h" diff --git a/Immortal/impch.h b/Immortal/impch.h index 25763445..cf024553 100644 --- a/Immortal/impch.h +++ b/Immortal/impch.h @@ -24,15 +24,6 @@ #include "Algorithm/LightVector.h" #include "Algorithm/Rotate.h" -#include "Audio/AudioRenderContext.h" -#include "Audio/AudioSource.h" -#include "Audio/Device.h" -#if defined(_WIN32) -#include "Audio/WASAPI.h" -#elif defined(__linux__) -#include "Audio/ALSA.h" -#endif - #include "FileSystem/FileSystem.h" #include "FileSystem/RF.h" #include "FileSystem/Stream.h"