Skip to content

Commit

Permalink
Merge pull request #17703 from hrydgard/sound-effects-mixer
Browse files Browse the repository at this point in the history 
Change the sound effects mixer (used for UI sounds) to mix on top of main output
  • Loading branch information
@hrydgard
hrydgard committed Jul 12, 2023
2 parents 408c6dc + f1ff127 commit 277790f
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 94 deletions.
1 change: 0 additions & 1 deletion Common/UI/Root.cpp
View file
Expand Up @@ -226,7 +226,6 @@ static KeyEventResult KeyEventToFocusMoves(const KeyInput &key) {
}

heldKeys.insert(hk);
INFO_LOG(SCECTRL, "focus move: %d", key.keyCode);
focusMoves.push_back(key.keyCode);
retval = KeyEventResult::ACCEPT;
}
Expand Down
10 changes: 8 additions & 2 deletions UI/AudioCommon.cpp
View file
@@ -1,13 +1,19 @@
#include "Common/System/System.h"
#include "Core/HW/StereoResampler.h" // TODO: doesn't belong in Core/HW...
#include "UI/AudioCommon.h"
#include "UI/BackgroundAudio.h"

StereoResampler g_resampler;

// numFrames is number of stereo frames.
// This is called from *outside* the emulator thread.
int __AudioMix(int16_t *outstereo, int numFrames, int sampleRate) {
return g_resampler.Mix(outstereo, numFrames, false, sampleRate);
int __AudioMix(int16_t *outStereo, int numFrames, int sampleRateHz) {
int validFrames = g_resampler.Mix(outStereo, numFrames, false, sampleRateHz);

// Mix sound effects on top.
g_BackgroundAudio.SFX().Mix(outStereo, validFrames, sampleRateHz);

return validFrames;
}

void System_AudioGetDebugStats(char *buf, size_t bufSize) {
Expand Down
155 changes: 90 additions & 65 deletions UI/BackgroundAudio.cpp
View file
Expand Up @@ -257,49 +257,6 @@ BackgroundAudio::~BackgroundAudio() {
delete[] buffer;
}

BackgroundAudio::Sample *BackgroundAudio::LoadSample(const std::string &path) {
size_t bytes;
uint8_t *data = g_VFS.ReadFile(path.c_str(), &bytes);
if (!data) {
return nullptr;
}

RIFFReader reader(data, (int)bytes);

WavData wave;
wave.Read(reader);

delete [] data;

if (wave.num_channels != 2 || wave.sample_rate != 44100 || wave.raw_bytes_per_frame != 4) {
ERROR_LOG(AUDIO, "Wave format not supported for mixer playback. Must be 16-bit raw stereo. '%s'", path.c_str());
return nullptr;
}

int16_t *samples = new int16_t[2 * wave.numFrames];
memcpy(samples, wave.raw_data, wave.numFrames * wave.raw_bytes_per_frame);

return new BackgroundAudio::Sample(samples, wave.numFrames);
}

void BackgroundAudio::LoadSamples() {
samples_.resize((size_t)UI::UISound::COUNT);
samples_[(size_t)UI::UISound::BACK] = std::unique_ptr<Sample>(LoadSample("sfx_back.wav"));
samples_[(size_t)UI::UISound::SELECT] = std::unique_ptr<Sample>(LoadSample("sfx_select.wav"));
samples_[(size_t)UI::UISound::CONFIRM] = std::unique_ptr<Sample>(LoadSample("sfx_confirm.wav"));
samples_[(size_t)UI::UISound::TOGGLE_ON] = std::unique_ptr<Sample>(LoadSample("sfx_toggle_on.wav"));
samples_[(size_t)UI::UISound::TOGGLE_OFF] = std::unique_ptr<Sample>(LoadSample("sfx_toggle_off.wav"));

UI::SetSoundCallback([](UI::UISound sound) {
g_BackgroundAudio.PlaySFX(sound);
});
}

void BackgroundAudio::PlaySFX(UI::UISound sfx) {
std::lock_guard<std::mutex> lock(mutex_);
plays_.push_back(PlayInstance{ sfx, 0, 64, false });
}

void BackgroundAudio::Clear(bool hard) {
if (!hard) {
fadingOut_ = true;
Expand Down Expand Up @@ -372,28 +329,6 @@ bool BackgroundAudio::Play() {
}
}

// Mix in menu sound effects. Terribly slow mixer but meh.
if (!plays_.empty()) {
for (int i = 0; i < sz * 2; i += 2) {
std::vector<PlayInstance>::iterator iter = plays_.begin();
while (iter != plays_.end()) {
PlayInstance inst = *iter;
auto sample = samples_[(int)inst.sound].get();
if (!sample || iter->offset >= sample->length_) {
iter->done = true;
iter = plays_.erase(iter);
} else {
if (!iter->done) {
buffer[i] += sample->data_[inst.offset * 2] * inst.volume >> 8;
buffer[i + 1] += sample->data_[inst.offset * 2 + 1] * inst.volume >> 8;
}
iter->offset++;
iter++;
}
}
}
}

System_AudioPushSamples(buffer, sz);

if (at3Reader_ && fadingOut_ && volume_ <= 0.0f) {
Expand Down Expand Up @@ -431,3 +366,93 @@ void BackgroundAudio::Update() {
sndLoadPending_ = false;
}
}

static inline int16_t Clamp16(int32_t sample) {
if (sample < -32767) return -32767;
if (sample > 32767) return 32767;
return sample;
}

void SoundEffectMixer::Mix(int16_t *buffer, int sz, int sampleRateHz) {
// Mix in menu sound effects. Terribly slow mixer but meh.
if (plays_.empty()) {
return;
}

for (std::vector<PlayInstance>::iterator iter = plays_.begin(); iter != plays_.end(); ) {
auto sample = samples_[(int)iter->sound].get();

int64_t rateOfSample = sample->rateInHz_;
int64_t stride = (rateOfSample << 32) / sampleRateHz;

for (int i = 0; i < sz * 2; i += 2) {
if (!sample || (iter->offset >> 32) >= sample->length_ - 2) {
iter->done = true;
break;
}

int wholeOffset = iter->offset >> 32;
int frac = (iter->offset >> 20) & 0xFFF; // Use a 12 bit fraction to get away with 32-bit multiplies

int interpolatedLeft = (sample->data_[wholeOffset * 2] * (0x1000 - frac) + sample->data_[(wholeOffset + 1) * 2] * frac) >> 12;
int interpolatedRight = (sample->data_[wholeOffset * 2 + 1] * (0x1000 - frac) + sample->data_[(wholeOffset + 1) * 2 + 1] * frac) >> 12;

// Clamping add on top per sample. Not great, we should be mixing at higher bitrate instead. Oh well.
int left = Clamp16(buffer[i] + (interpolatedLeft * iter->volume >> 8));
int right = Clamp16(buffer[i + 1] + (interpolatedRight * iter->volume >> 8));

buffer[i] = left;
buffer[i + 1] = right;

iter->offset += stride;
}

if (iter->done) {
iter = plays_.erase(iter);
} else {
iter++;
}
}
}

void SoundEffectMixer::Play(UI::UISound sfx) {
std::lock_guard<std::mutex> guard(mutex_);
plays_.push_back(PlayInstance{ sfx, 0, 64, false });
}

Sample *SoundEffectMixer::LoadSample(const std::string &path) {
size_t bytes;
uint8_t *data = g_VFS.ReadFile(path.c_str(), &bytes);
if (!data) {
return nullptr;
}

RIFFReader reader(data, (int)bytes);

WavData wave;
wave.Read(reader);

delete[] data;

if (wave.num_channels != 2 || wave.raw_bytes_per_frame != 4) {
ERROR_LOG(AUDIO, "Wave format not supported for mixer playback. Must be 16-bit raw stereo. '%s'", path.c_str());
return nullptr;
}

int16_t *samples = new int16_t[2 * wave.numFrames];
memcpy(samples, wave.raw_data, wave.numFrames * wave.raw_bytes_per_frame);
return new Sample(samples, wave.numFrames, wave.sample_rate);
}

void SoundEffectMixer::LoadSamples() {
samples_.resize((size_t)UI::UISound::COUNT);
samples_[(size_t)UI::UISound::BACK] = std::unique_ptr<Sample>(LoadSample("sfx_back.wav"));
samples_[(size_t)UI::UISound::SELECT] = std::unique_ptr<Sample>(LoadSample("sfx_select.wav"));
samples_[(size_t)UI::UISound::CONFIRM] = std::unique_ptr<Sample>(LoadSample("sfx_confirm.wav"));
samples_[(size_t)UI::UISound::TOGGLE_ON] = std::unique_ptr<Sample>(LoadSample("sfx_toggle_on.wav"));
samples_[(size_t)UI::UISound::TOGGLE_OFF] = std::unique_ptr<Sample>(LoadSample("sfx_toggle_off.wav"));

UI::SetSoundCallback([](UI::UISound sound) {
g_BackgroundAudio.SFX().Play(sound);
});
}
63 changes: 38 additions & 25 deletions UI/BackgroundAudio.h
View file
Expand Up @@ -10,6 +10,39 @@

class AT3PlusReader;

struct Sample {
// data must be new-ed.
Sample(int16_t *data, int length, int rateInHz) : data_(data), length_(length), rateInHz_(rateInHz) {}
~Sample() {
delete[] data_;
}
int16_t *data_;
int length_; // stereo samples.
int rateInHz_; // sampleRate
};

// Mixer for things played on top of everything.
class SoundEffectMixer {
public:
static Sample *LoadSample(const std::string &path);
void LoadSamples();

void Mix(int16_t *buffer, int sz, int sampleRateHz);
void Play(UI::UISound sfx);

std::vector<std::unique_ptr<Sample>> samples_;

struct PlayInstance {
UI::UISound sound;
int64_t offset; // 32.32 fixed point
int volume; // 0..255
bool done;
};

std::mutex mutex_;
std::vector<PlayInstance> plays_;
};

class BackgroundAudio {
public:
BackgroundAudio();
Expand All @@ -19,8 +52,9 @@ class BackgroundAudio {
void Update();
bool Play();

void LoadSamples();
void PlaySFX(UI::UISound sfx);
SoundEffectMixer &SFX() {
return sfxMixer_;
}

private:
void Clear(bool hard);
Expand All @@ -33,35 +67,14 @@ class BackgroundAudio {
Path bgGamePath_;
std::atomic<bool> sndLoadPending_;
int playbackOffset_ = 0;
AT3PlusReader *at3Reader_;
AT3PlusReader *at3Reader_ = nullptr;
double gameLastChanged_ = 0.0;
double lastPlaybackTime_ = 0.0;
int *buffer = nullptr;
bool fadingOut_ = true;
float volume_ = 0.0f;
float delta_ = -0.0001f;

struct PlayInstance {
UI::UISound sound;
int offset;
int volume; // 0..255
bool done;
};

struct Sample {
// data must be new-ed.
Sample(int16_t *data, int length) : data_(data), length_(length) {}
~Sample() {
delete[] data_;
}
int16_t *data_;
int length_; // stereo samples.
};

static Sample *LoadSample(const std::string &path);

std::vector<PlayInstance> plays_;
std::vector<std::unique_ptr<Sample>> samples_;
SoundEffectMixer sfxMixer_;
};

extern BackgroundAudio g_BackgroundAudio;
2 changes: 1 addition & 1 deletion UI/NativeApp.cpp
View file
Expand Up @@ -763,7 +763,7 @@ void NativeInit(int argc, const char *argv[], const char *savegame_dir, const ch
#endif

// TODO: Load these in the background instead of synchronously.
g_BackgroundAudio.LoadSamples();
g_BackgroundAudio.SFX().LoadSamples();

if (!boot_filename.empty() && stateToLoad.Valid()) {
SaveState::Load(stateToLoad, -1, [](SaveState::Status status, const std::string &message, void *) {
Expand Down

0 comments on commit 277790f

Please sign in to comment.