how do i use whisper STT(Speach to Text) from the system mic?

[pheonixfirewingz]

I wanted to recreate an old Siri-like AI I built in high school. The original was written in C# and relied on Windows APIs, which makes it unsuitable for cross-platform use. So, I decided to use Whisper for the speech-to-text part, but I’m having trouble getting it to work with a live microphone input.

Right now, I’m focusing on Windows and plan to add Linux support later. The official examples aren’t very clear to me, and I’m struggling to figure out what I’m doing wrong. Could someone help me understand how to properly integrate Whisper with a live mic stream?

WhisperMicStreamer.h

#pragma once

#include <atomic>
#include <thread>
#include <vector>
#include <memory>
#include <functional>
#include <mutex>
#include <string>
#include <STT/Mic/IMic.h>
#include <whisper.h>

namespace LunaAI::STT {

    class WhisperMicStreamer {
    public:
        WhisperMicStreamer();
        ~WhisperMicStreamer();

        bool Start();
        void Stop();
        bool IsRunning() const;

        // Set a callback to receive transcribed text
        void SetTranscriptionCallback(std::function<void(const std::string&)> callback);

    private:
        void StreamLoop();

        whisper_context* whisperCtx = nullptr;
        std::thread streamThread;
        std::atomic<bool> running = false;
        std::unique_ptr<Audio::IMic> mic;

        std::function<void(const std::string&)> transcriptionCallback;
        std::mutex callbackMutex;

        std::vector<float> pcmBuffer;
        std::mutex pcmMutex;
    };

} // namespace LunaAI::STT

WhisperMicStreamer.cpp

#include <STT/WhisperMicStreamer.h>

#if defined(_WIN32) || defined(_WIN64)
#include <STT/Mic/Mic_Windows.h>
#elif defined(__linux__) || defined(__APPLE__)
#include <STT/Mic/Mic_Linux.h>
#else
#error "Unsupported platform"
#endif

#include <Config.h>
#include <iostream>
#include <chrono>
#include <thread>

constexpr int SAMPLE_RATE = 16000;
constexpr int CHANNELS = 1;
const char* MODEL = PROJ_ROOT "/models/ggml-base.en.bin";

namespace LunaAI::STT {

    WhisperMicStreamer::WhisperMicStreamer() {
        whisperCtx = whisper_init_from_file(MODEL);
#if defined(_WIN32) || defined(_WIN64)
        mic = std::make_unique<Audio::MicWindows>();
#elif defined(__linux__) || defined(__APPLE__)
        mic = std::make_unique<Audio::MicLinux>();
#endif
    }

    WhisperMicStreamer::~WhisperMicStreamer() {
        Stop();
        if (whisperCtx) {
            whisper_free(whisperCtx);
            whisperCtx = nullptr;
        }
    }

    bool WhisperMicStreamer::Start() {
        if (running.load()) return false;

        if (!mic->Initialize(SAMPLE_RATE, CHANNELS,
            [this](const int16_t* samples, size_t sampleCount) {
                if (!running.load()) return;

                std::lock_guard lock(pcmMutex);
                for (size_t i = 0; i < sampleCount; ++i) {
                    pcmBuffer.push_back(samples[i] / 32768.0f);
                }
            })) {
            std::cerr << "Failed to initialize mic\n";
            return false;
        }

        running.store(true);
        mic->Start();
        streamThread = std::thread(&WhisperMicStreamer::StreamLoop, this);
        return true;
    }

    void WhisperMicStreamer::Stop() {
        if (!running.load()) return;

        running.store(false);
        mic->Stop();
        mic->Shutdown();

        if (streamThread.joinable())
            streamThread.join();
    }

    bool WhisperMicStreamer::IsRunning() const {
        return running.load();
    }

    void WhisperMicStreamer::SetTranscriptionCallback(std::function<void(const std::string&)> callback) {
        std::lock_guard<std::mutex> lock(callbackMutex);
        transcriptionCallback = std::move(callback);
    }

    void WhisperMicStreamer::StreamLoop() {
        while (running.load()) {
            std::vector<float> localBuffer;

            {
                std::lock_guard lock(pcmMutex);
                if (pcmBuffer.size() >= 1600) {
                    localBuffer.swap(pcmBuffer); // take audio for processing
                }
            }

            if (localBuffer.size() >= 1600) {
                whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
                params.print_realtime = false;
                params.print_progress = false;
                params.print_special = false;
                params.translate = false;
                params.max_tokens = 0;

                int ret = whisper_full(whisperCtx, params, localBuffer.data(), (int)localBuffer.size());
                if (ret != 0) {
                    std::cerr << "Whisper inference error: " << ret << "\n";
                }
                else {
                    const int n_segments = whisper_full_n_segments(whisperCtx);
                    for (int i = 0; i < n_segments; i++) {
                        const char* text = whisper_full_get_segment_text(whisperCtx, i);
                        std::lock_guard<std::mutex> lock(callbackMutex);
                        if (transcriptionCallback)
                            transcriptionCallback(text);
                    }
                }
            }

            std::this_thread::sleep_for(std::chrono::milliseconds(100));
        }
    }

} // namespace LunaAI::STT

Mic_windows.cpp

#if defined(_WIN32) || defined(_WIN64)
#include <STT/Mic/Mic_Windows.h>
#include <stdexcept>
#include <cstring>
namespace LunaAI::Audio {

	MicWindows::MicWindows() {}

	MicWindows::~MicWindows() {
		Shutdown();
	}

	bool MicWindows::Initialize(int sampleRate_, int channels_, AudioCallback callback_) {
		sampleRate = sampleRate_;
		channels = channels_;
		callback = std::move(callback_);

		WAVEFORMATEX format = {};
		format.wFormatTag = WAVE_FORMAT_PCM;
		format.nChannels = static_cast<WORD>(channels);
		format.nSamplesPerSec = sampleRate;
		format.wBitsPerSample = 16;
		format.nBlockAlign = format.nChannels * format.wBitsPerSample / 8;
		format.nAvgBytesPerSec = format.nSamplesPerSec * format.nBlockAlign;

		MMRESULT result = waveInOpen(&hWaveIn, WAVE_MAPPER, &format,
			reinterpret_cast<DWORD_PTR>(&WaveInProc),
			reinterpret_cast<DWORD_PTR>(this),
			CALLBACK_FUNCTION);

		if (result != MMSYSERR_NOERROR)
			return false;

		const size_t bufferSize = (sampleRate * channels) / 10; // 100ms buffer
		const int numBuffers = 2;

		buffers.resize(numBuffers);
		rawBuffers.resize(numBuffers);

		for (int i = 0; i < numBuffers; ++i) {
			rawBuffers[i].resize(bufferSize);
			WAVEHDR& hdr = buffers[i];

			hdr.lpData = reinterpret_cast<LPSTR>(rawBuffers[i].data());
			hdr.dwBufferLength = static_cast<DWORD>(bufferSize * sizeof(int16_t));
			hdr.dwFlags = 0;
			hdr.dwLoops = 0;

			waveInPrepareHeader(hWaveIn, &hdr, sizeof(WAVEHDR));
			waveInAddBuffer(hWaveIn, &hdr, sizeof(WAVEHDR));
		}

		return true;
	}

	void MicWindows::Start() {
		if (!running) {
			running = true;
			waveInStart(hWaveIn);
		}
	}

	void MicWindows::Stop() {
		if (running) {
			waveInStop(hWaveIn);
			running = false;
		}
	}

	void MicWindows::Shutdown() {
		if (hWaveIn) {
			Stop();

			for (auto& hdr : buffers) {
				waveInUnprepareHeader(hWaveIn, &hdr, sizeof(WAVEHDR));
			}

			waveInClose(hWaveIn);
			hWaveIn = nullptr;

			buffers.clear();
			rawBuffers.clear();
		}
	}

	void CALLBACK MicWindows::WaveInProc(HWAVEIN hwi, UINT uMsg, DWORD_PTR dwInstance, DWORD_PTR dwParam1, DWORD_PTR) {
		if (uMsg == WIM_DATA) {
			auto* self = reinterpret_cast<MicWindows*>(dwInstance);
			auto* hdr = reinterpret_cast<WAVEHDR*>(dwParam1);
			self->ProcessAudio(hdr);
		}
	}

	void MicWindows::ProcessAudio(WAVEHDR* header) {
		if (callback && running) {
			int16_t* samples = reinterpret_cast<int16_t*>(header->lpData);
			size_t numSamples = header->dwBytesRecorded / sizeof(int16_t);
			callback(samples, numSamples);
		}

		if (running) {
			waveInAddBuffer(hWaveIn, header, sizeof(WAVEHDR));
		}
	}

} // namespace LunaAI::Audio

#endif // _WIN32

[RndyP]

What is going wrong? Is your callback being called?

[pheonixfirewingz]

The whisper is an STT integrated daemon that runs always, collecting the mic data and running it through whisper and then sending the text to the callback "transcriptionCallback" to be processed, at least that is the intended functionality. On the other side, it will be sent to another thread to handle command detection or sent to an LLM for conversation,