-
Notifications
You must be signed in to change notification settings - Fork 6.6k
/
audio_low_latency_input_win.cc
1683 lines (1492 loc) · 66.9 KB
/
audio_low_latency_input_win.cc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2012 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/audio/win/audio_low_latency_input_win.h"
#include <objbase.h>
#include <propkey.h>
#include <windows.devices.enumeration.h>
#include <windows.media.devices.h>
#include <algorithm>
#include <cmath>
#include <memory>
#include <utility>
#include "base/logging.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/strings/stringprintf.h"
#include "base/strings/utf_string_conversions.h"
#include "base/trace_event/common/trace_event_common.h"
#include "base/trace_event/trace_event.h"
#include "base/win/core_winrt_util.h"
#include "base/win/scoped_propvariant.h"
#include "base/win/scoped_variant.h"
#include "base/win/vector.h"
#include "base/win/windows_version.h"
#include "media/audio/audio_device_description.h"
#include "media/audio/audio_features.h"
#include "media/audio/win/avrt_wrapper_win.h"
#include "media/audio/win/core_audio_util_win.h"
#include "media/base/audio_block_fifo.h"
#include "media/base/audio_bus.h"
#include "media/base/audio_timestamp_helper.h"
#include "media/base/channel_layout.h"
#include "media/base/limits.h"
#include "media/base/media_switches.h"
#include "media/base/timestamp_constants.h"
using ABI::Windows::Foundation::Collections::IVectorView;
using ABI::Windows::Media::Devices::IMediaDeviceStatics;
using ABI::Windows::Media::Effects::IAudioCaptureEffectsManager;
using ABI::Windows::Media::Effects::IAudioEffectsManagerStatics;
using base::win::GetActivationFactory;
using base::win::ScopedCoMem;
using base::win::ScopedCOMInitializer;
using base::win::ScopedHString;
using Microsoft::WRL::ComPtr;
namespace media {
namespace {
constexpr char kUwpDeviceIdPrefix[] = "\\\\?\\SWD#MMDEVAPI#";
constexpr uint32_t KSAUDIO_SPEAKER_UNSUPPORTED = 0;
// Converts a COM error into a human-readable string.
std::string ErrorToString(HRESULT hresult) {
return CoreAudioUtil::ErrorToString(hresult);
}
// Errors when initializing the audio client related to the audio format. Split
// by whether we're using format conversion or not. Used for reporting stats -
// do not renumber entries.
enum FormatRelatedInitError {
kUnsupportedFormat = 0,
kUnsupportedFormatWithFormatConversion = 1,
kInvalidArgument = 2,
kInvalidArgumentWithFormatConversion = 3,
kCount
};
bool IsSupportedFormatForConversion(WAVEFORMATEXTENSIBLE* format_ex) {
WAVEFORMATEX* format = &format_ex->Format;
if (format->nSamplesPerSec < limits::kMinSampleRate ||
format->nSamplesPerSec > limits::kMaxSampleRate) {
return false;
}
switch (format->wBitsPerSample) {
case 8:
case 16:
case 32:
break;
default:
return false;
}
if (GuessChannelLayout(format->nChannels) == CHANNEL_LAYOUT_UNSUPPORTED) {
LOG(ERROR) << "Hardware configuration not supported for audio conversion";
return false;
}
return true;
}
// Converts ChannelLayout to Microsoft's channel configuration but only discrete
// and up to stereo is supported currently. All other multi-channel layouts
// return KSAUDIO_SPEAKER_UNSUPPORTED.
ChannelConfig ChannelLayoutToChannelConfig(ChannelLayout layout) {
switch (layout) {
case CHANNEL_LAYOUT_DISCRETE:
return KSAUDIO_SPEAKER_DIRECTOUT;
case CHANNEL_LAYOUT_MONO:
return KSAUDIO_SPEAKER_MONO;
case CHANNEL_LAYOUT_STEREO:
return KSAUDIO_SPEAKER_STEREO;
default:
LOG(WARNING) << "Unsupported channel layout: " << layout;
// KSAUDIO_SPEAKER_UNSUPPORTED equals 0 and corresponds to "no specific
// channel order".
return KSAUDIO_SPEAKER_UNSUPPORTED;
}
}
const char* StreamOpenResultToString(
WASAPIAudioInputStream::StreamOpenResult result) {
switch (result) {
case WASAPIAudioInputStream::OPEN_RESULT_OK:
return "OK";
case WASAPIAudioInputStream::OPEN_RESULT_CREATE_INSTANCE:
return "CREATE_INSTANCE";
case WASAPIAudioInputStream::OPEN_RESULT_NO_ENDPOINT:
return "NO_ENDPOINT";
case WASAPIAudioInputStream::OPEN_RESULT_NO_STATE:
return "NO_STATE";
case WASAPIAudioInputStream::OPEN_RESULT_DEVICE_NOT_ACTIVE:
return "DEVICE_NOT_ACTIVE";
case WASAPIAudioInputStream::OPEN_RESULT_ACTIVATION_FAILED:
return "ACTIVATION_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_FORMAT_NOT_SUPPORTED:
return "FORMAT_NOT_SUPPORTED";
case WASAPIAudioInputStream::OPEN_RESULT_AUDIO_CLIENT_INIT_FAILED:
return "AUDIO_CLIENT_INIT_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_GET_BUFFER_SIZE_FAILED:
return "GET_BUFFER_SIZE_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_LOOPBACK_ACTIVATE_FAILED:
return "LOOPBACK_ACTIVATE_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_LOOPBACK_INIT_FAILED:
return "LOOPBACK_INIT_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_SET_EVENT_HANDLE:
return "SET_EVENT_HANDLE";
case WASAPIAudioInputStream::OPEN_RESULT_NO_CAPTURE_CLIENT:
return "NO_CAPTURE_CLIENT";
case WASAPIAudioInputStream::OPEN_RESULT_NO_AUDIO_VOLUME:
return "NO_AUDIO_VOLUME";
case WASAPIAudioInputStream::OPEN_RESULT_OK_WITH_RESAMPLING:
return "OK_WITH_RESAMPLING";
}
return "UNKNOWN";
}
const char* EffectTypeToString(
ABI::Windows::Media::Effects::AudioEffectType type) {
switch (type) {
case ABI::Windows::Media::Effects::AudioEffectType_Other:
return "Other/None";
case ABI::Windows::Media::Effects::AudioEffectType_AcousticEchoCancellation:
return "AcousticEchoCancellation";
case ABI::Windows::Media::Effects::AudioEffectType_NoiseSuppression:
return "NoiseSuppression";
case ABI::Windows::Media::Effects::AudioEffectType_AutomaticGainControl:
return "AutomaticGainControl";
case ABI::Windows::Media::Effects::AudioEffectType_BeamForming:
return "BeamForming";
case ABI::Windows::Media::Effects::AudioEffectType_ConstantToneRemoval:
return "ConstantToneRemoval";
case ABI::Windows::Media::Effects::AudioEffectType_Equalizer:
return "Equalizer";
case ABI::Windows::Media::Effects::AudioEffectType_LoudnessEqualizer:
return "LoudnessEqualizer";
case ABI::Windows::Media::Effects::AudioEffectType_BassBoost:
return "BassBoost";
case ABI::Windows::Media::Effects::AudioEffectType_VirtualSurround:
return "VirtualSurround";
case ABI::Windows::Media::Effects::AudioEffectType_VirtualHeadphones:
return "VirtualHeadphones";
case ABI::Windows::Media::Effects::AudioEffectType_SpeakerFill:
return "SpeakerFill";
case ABI::Windows::Media::Effects::AudioEffectType_RoomCorrection:
return "RoomCorrection";
case ABI::Windows::Media::Effects::AudioEffectType_BassManagement:
return "BassManagement";
case ABI::Windows::Media::Effects::AudioEffectType_EnvironmentalEffects:
return "EnvironmentalEffects";
case ABI::Windows::Media::Effects::AudioEffectType_SpeakerProtection:
return "SpeakerProtection";
case ABI::Windows::Media::Effects::AudioEffectType_SpeakerCompensation:
return "SpeakerCompensation";
case ABI::Windows::Media::Effects::AudioEffectType_DynamicRangeCompression:
return "DynamicRangeCompression";
case ABI::Windows::Media::Effects::AudioEffectType_FarFieldBeamForming:
return "FarFieldBeamForming";
case ABI::Windows::Media::Effects::AudioEffectType_DeepNoiseSuppression:
return "DeepNoiseSuppression";
}
return "Unknown";
}
bool VariantBoolToBool(VARIANT_BOOL var_bool) {
switch (var_bool) {
case VARIANT_TRUE:
return true;
case VARIANT_FALSE:
return false;
}
LOG(ERROR) << "Invalid VARIANT_BOOL type";
return false;
}
std::string GetOpenLogString(WASAPIAudioInputStream::StreamOpenResult result,
HRESULT hr,
WAVEFORMATEXTENSIBLE input_format,
WAVEFORMATEX output_format) {
return base::StringPrintf(
"WAIS::Open => (ERROR: result=%s, hresult=%#lx, input_format=[%s], "
"output_format=[%s])",
StreamOpenResultToString(result), hr,
CoreAudioUtil::WaveFormatToString(&input_format).c_str(),
CoreAudioUtil::WaveFormatToString(&output_format).c_str());
}
bool InitializeUWPSupport() {
// Place the actual body of the initialization in a lambda and store the
// result as a static since we don't expect this result to change between
// runs.
static const bool initialization_result = []() {
// Windows.Media.Effects and Windows.Media.Devices requires Windows 10 build
// 10.0.10240.0.
DCHECK_GE(base::win::OSInfo::GetInstance()->version_number().build, 10240u);
return true;
}();
return initialization_result;
}
} // namespace
// Counts how often an OS capture callback reports a data discontinuity and logs
// it as a UMA histogram.
class WASAPIAudioInputStream::DataDiscontinuityReporter {
public:
// Logs once every 10s, assuming 10ms buffers.
constexpr static int kCallbacksPerLogPeriod = 1000;
DataDiscontinuityReporter() {}
int GetLongTermDiscontinuityCountAndReset() {
int long_term_count = data_discontinuity_long_term_count_;
callback_count_ = 0;
data_discontinuity_short_term_count_ = 0;
data_discontinuity_long_term_count_ = 0;
return long_term_count;
}
void Log(bool observed_data_discontinuity) {
++callback_count_;
if (observed_data_discontinuity) {
++data_discontinuity_short_term_count_;
++data_discontinuity_long_term_count_;
}
if (callback_count_ % kCallbacksPerLogPeriod)
return;
// TODO(https://crbug.com/825744): It can be possible to replace
// "Media.Audio.Capture.Glitches2" with this new (simplified) metric
// instead.
base::UmaHistogramCounts1000("Media.Audio.Capture.Win.Glitches2",
data_discontinuity_short_term_count_);
data_discontinuity_short_term_count_ = 0;
}
private:
int callback_count_ = 0;
int data_discontinuity_short_term_count_ = 0;
int data_discontinuity_long_term_count_ = 0;
};
WASAPIAudioInputStream::WASAPIAudioInputStream(
AudioManagerWin* manager,
const AudioParameters& params,
const std::string& device_id,
AudioManager::LogCallback log_callback)
: manager_(manager),
glitch_reporter_(SystemGlitchReporter::StreamType::kCapture),
peak_detector_(base::BindRepeating(&AudioManager::TraceAmplitudePeak,
base::Unretained(manager_),
/*trace_start=*/true)),
data_discontinuity_reporter_(
std::make_unique<DataDiscontinuityReporter>()),
device_id_(device_id),
log_callback_(std::move(log_callback)) {
DCHECK(manager_);
DCHECK(!device_id_.empty());
DCHECK(!log_callback_.is_null());
DCHECK_LE(params.channels(), 2);
DCHECK(params.channel_layout() == CHANNEL_LAYOUT_MONO ||
params.channel_layout() == CHANNEL_LAYOUT_STEREO ||
params.channel_layout() == CHANNEL_LAYOUT_DISCRETE);
SendLogMessage("%s({device_id=%s}, {params=[%s]})", __func__,
device_id.c_str(), params.AsHumanReadableString().c_str());
// Load the Avrt DLL if not already loaded. Required to support MMCSS.
bool avrt_init = avrt::Initialize();
if (!avrt_init)
SendLogMessage("%s => (WARNING: failed to load Avrt.dll)", __func__);
const SampleFormat kSampleFormat = kSampleFormatS16;
// The clients asks for an input stream specified by |params|. Start by
// setting up an input device format according to the same specification.
// If all goes well during the upcoming initialization, this format will not
// change. However, under some circumstances, minor changes can be required
// to fit the current input audio device. If so, a FIFO and/or and audio
// converter might be needed to ensure that the output format of this stream
// matches what the client asks for.
WAVEFORMATEX* format = &input_format_.Format;
format->wFormatTag = WAVE_FORMAT_EXTENSIBLE;
format->nChannels = params.channels();
format->nSamplesPerSec = params.sample_rate();
format->wBitsPerSample = SampleFormatToBitsPerChannel(kSampleFormat);
format->nBlockAlign = (format->wBitsPerSample / 8) * format->nChannels;
format->nAvgBytesPerSec = format->nSamplesPerSec * format->nBlockAlign;
// Add the parts which are unique to WAVE_FORMAT_EXTENSIBLE which can be
// required in combination with e.g. multi-channel microphone arrays.
format->cbSize = sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX);
input_format_.Samples.wValidBitsPerSample = format->wBitsPerSample;
input_format_.dwChannelMask =
ChannelLayoutToChannelConfig(params.channel_layout());
input_format_.SubFormat = KSDATAFORMAT_SUBTYPE_PCM;
SendLogMessage("%s => (audio engine format=[%s])", __func__,
CoreAudioUtil::WaveFormatToString(&input_format_).c_str());
// Set up the fixed output format based on |params|. Will not be changed and
// does not required an extended wave format structure since any multi-channel
// input will be converted to stereo.
output_format_.wFormatTag = WAVE_FORMAT_PCM;
output_format_.nChannels = format->nChannels;
output_format_.nSamplesPerSec = format->nSamplesPerSec;
output_format_.wBitsPerSample = format->wBitsPerSample;
output_format_.nBlockAlign = format->nBlockAlign;
output_format_.nAvgBytesPerSec = format->nAvgBytesPerSec;
output_format_.cbSize = 0;
SendLogMessage("%s => (audio sink format=[%s])", __func__,
CoreAudioUtil::WaveFormatToString(&output_format_).c_str());
// Size in bytes of each audio frame.
frame_size_bytes_ = format->nBlockAlign;
// Store size of audio packets which we expect to get from the audio
// endpoint device in each capture event.
packet_size_bytes_ = params.GetBytesPerBuffer(kSampleFormat);
packet_size_frames_ = packet_size_bytes_ / format->nBlockAlign;
SendLogMessage(
"%s => (packet size=[%zu bytes/%zu audio frames/%.3f milliseconds])",
__func__, packet_size_bytes_, packet_size_frames_,
params.GetBufferDuration().InMillisecondsF());
// All events are auto-reset events and non-signaled initially.
// Create the event which the audio engine will signal each time
// a buffer becomes ready to be processed by the client.
audio_samples_ready_event_.Set(CreateEvent(NULL, FALSE, FALSE, NULL));
DCHECK(audio_samples_ready_event_.IsValid());
// Create the event which will be set in Stop() when capturing shall stop.
stop_capture_event_.Set(CreateEvent(NULL, FALSE, FALSE, NULL));
DCHECK(stop_capture_event_.IsValid());
}
WASAPIAudioInputStream::~WASAPIAudioInputStream() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
}
AudioInputStream::OpenOutcome WASAPIAudioInputStream::Open() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
SendLogMessage("%s([opened=%s])", __func__, opened_ ? "true" : "false");
if (opened_) {
return OpenOutcome::kAlreadyOpen;
}
// Obtain a reference to the IMMDevice interface of the capturing device with
// the specified unique identifier or role which was set at construction.
HRESULT hr = SetCaptureDevice();
if (FAILED(hr)) {
ReportOpenResult(hr);
return OpenOutcome::kFailed;
}
// Check if raw audio processing is supported for the selected capture device.
raw_processing_supported_ = RawProcessingSupported();
if (raw_processing_supported_ &&
!AudioDeviceDescription::IsLoopbackDevice(device_id_) &&
InitializeUWPSupport()) {
// Retrieve a unique identifier of the selected audio device but in a
// format which can be used by UWP (or Core WinRT) APIs. It can then be
// utilized in combination with the Windows.Media.Effects UWP API to
// discover the audio processing chain on a device.
std::string uwp_device_id = GetUWPDeviceId();
if (!uwp_device_id.empty()) {
// For the selected device, generate two lists of enabled audio effects
// and store them in |default_effect_types_| and |raw_effect_types_|.
// Default corresponds to "Normal audio signal processing" and Raw is for
// "Minimal audio signal processing". These two lists are used for UMA
// stats when the stream is closed.
GetAudioCaptureEffects(uwp_device_id);
}
}
use_fake_audio_capture_timestamps_ =
base::FeatureList::IsEnabled(media::kUseFakeAudioCaptureTimestamps);
if (use_fake_audio_capture_timestamps_) {
SendLogMessage("%s => (WARNING: capture timestamps will be fake)",
__func__);
}
// Obtain an IAudioClient interface which enables us to create and initialize
// an audio stream between an audio application and the audio engine.
hr = endpoint_device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr,
&audio_client_);
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_ACTIVATION_FAILED;
ReportOpenResult(hr);
return OpenOutcome::kFailed;
}
// Raw audio capture suppresses processing that down mixes e.g. a microphone
// array into a supported format and instead exposes the device's native
// format. Chrome only supports a maximum number of input channels given by
// media::kMaxConcurrentChannels. Therefore, one additional test is needed
// before stating that raw audio processing can be supported.
// Failure will not prevent opening but the method must succeed to be able to
// select raw input capture mode.
WORD audio_engine_channels = 0;
hr = GetAudioEngineNumChannels(&audio_engine_channels);
// Attempt to enable communications category and raw capture mode on the audio
// stream. Ignoring return value since the method logs its own error messages
// and it should be OK to continue opening the stream even after a failure.
if (base::FeatureList::IsEnabled(media::kWasapiRawAudioCapture) &&
raw_processing_supported_ &&
!AudioDeviceDescription::IsLoopbackDevice(device_id_) && SUCCEEDED(hr)) {
SetCommunicationsCategoryAndMaybeRawCaptureMode(audio_engine_channels);
}
// Verify that the selected audio endpoint supports the specified format
// set during construction and using the specified client properties.
hr = S_OK;
if (!DesiredFormatIsSupported(&hr)) {
open_result_ = OPEN_RESULT_FORMAT_NOT_SUPPORTED;
ReportOpenResult(hr);
return OpenOutcome::kFailed;
}
// Initialize the audio stream between the client and the device using
// shared mode and a lowest possible glitch-free latency.
hr = InitializeAudioEngine();
if (SUCCEEDED(hr) && converter_)
open_result_ = OPEN_RESULT_OK_WITH_RESAMPLING;
ReportOpenResult(hr); // Report before we assign a value to |opened_|.
opened_ = SUCCEEDED(hr);
if (opened_) {
return OpenOutcome::kSuccess;
}
switch (hr) {
case E_ACCESSDENIED:
return OpenOutcome::kFailedSystemPermissions;
case AUDCLNT_E_DEVICE_IN_USE:
return OpenOutcome::kFailedInUse;
default:
return OpenOutcome::kFailed;
}
}
void WASAPIAudioInputStream::Start(AudioInputCallback* callback) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(callback);
SendLogMessage("%s([opened=%s, started=%s])", __func__,
opened_ ? "true" : "false", started_ ? "true" : "false");
if (!opened_)
return;
if (started_)
return;
// Check if the master volume level of the opened audio session is set to
// zero and store the information for a UMA histogram generated in Stop().
// Valid volume levels are in the range 0.0 to 1.0.
// See http://crbug.com/1014443 for details why this is needed.
if (GetVolume() == 0.0) {
SendLogMessage("%s => (WARNING: Input audio session starts at zero volume)",
__func__);
audio_session_starts_at_zero_volume_ = true;
}
if (device_id_ == AudioDeviceDescription::kLoopbackWithMuteDeviceId &&
system_audio_volume_) {
BOOL muted = false;
system_audio_volume_->GetMute(&muted);
// If the system audio is muted at the time of capturing, then no need to
// mute it again, and later we do not unmute system audio when stopping
// capturing.
if (!muted) {
system_audio_volume_->SetMute(true, nullptr);
mute_done_ = true;
}
}
DCHECK(!sink_);
sink_ = callback;
// Starts periodic AGC microphone measurements if the AGC has been enabled
// using SetAutomaticGainControl().
StartAgc();
// Create and start the thread that will drive the capturing by waiting for
// capture events.
DCHECK(!capture_thread_.get());
capture_thread_ = std::make_unique<base::DelegateSimpleThread>(
this, "wasapi_capture_thread",
base::SimpleThread::Options(base::ThreadType::kRealtimeAudio));
capture_thread_->Start();
// Start streaming data between the endpoint buffer and the audio engine.
HRESULT hr = audio_client_->Start();
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: IAudioClient::Start=[%s])", __func__,
ErrorToString(hr).c_str());
}
if (SUCCEEDED(hr) && audio_render_client_for_loopback_.Get()) {
hr = audio_render_client_for_loopback_->Start();
if (FAILED(hr))
SendLogMessage("%s => (ERROR: IAudioClient::Start=[%s] (loopback))",
__func__, ErrorToString(hr).c_str());
}
started_ = SUCCEEDED(hr);
}
void WASAPIAudioInputStream::Stop() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
SendLogMessage("%s([started=%s])", __func__, started_ ? "true" : "false");
if (!started_)
return;
// Only upload UMA histogram for the case when AGC is enabled, i.e., for
// WebRTC based audio input streams.
const bool add_uma_histogram = GetAutomaticGainControl();
// We have muted system audio for capturing, so we need to unmute it when
// capturing stops.
if (device_id_ == AudioDeviceDescription::kLoopbackWithMuteDeviceId &&
mute_done_) {
DCHECK(system_audio_volume_);
if (system_audio_volume_) {
system_audio_volume_->SetMute(false, nullptr);
mute_done_ = false;
}
}
// Stops periodic AGC microphone measurements.
StopAgc();
// Shut down the capture thread.
if (stop_capture_event_.IsValid()) {
SetEvent(stop_capture_event_.Get());
}
// Stop the input audio streaming.
HRESULT hr = audio_client_->Stop();
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: IAudioClient::Stop=[%s])", __func__,
ErrorToString(hr).c_str());
}
// Wait until the thread completes and perform cleanup.
if (capture_thread_) {
SetEvent(stop_capture_event_.Get());
capture_thread_->Join();
capture_thread_.reset();
}
// Upload UMA histogram to track down possible issue that can lead to a
// "no audio" state. See http://crbug.com/1014443.
if (add_uma_histogram) {
base::UmaHistogramBoolean("Media.Audio.InputVolumeStartsAtZeroWin",
audio_session_starts_at_zero_volume_);
audio_session_starts_at_zero_volume_ = false;
}
SendLogMessage(
"%s => (timestamp(n)-timestamp(n-1)=[min: %.3f msec, max: %.3f msec])",
__func__, min_timestamp_diff_.InMillisecondsF(),
max_timestamp_diff_.InMillisecondsF());
started_ = false;
sink_ = nullptr;
}
void WASAPIAudioInputStream::Close() {
SendLogMessage("%s()", __func__);
// It is valid to call Close() before calling open or Start().
// It is also valid to call Close() after Start() has been called.
Stop();
// Only upload UMA histogram for the case when AGC is enabled, i.e., for
// WebRTC based audio input streams.
if (GetAutomaticGainControl()) {
// Upload UMA histogram to track if the capture device supported raw audio
// capture or not. See https://crbug.com/1133643.
base::UmaHistogramBoolean("Media.Audio.RawProcessingSupportedWin",
raw_processing_supported_);
// These UMAs are deprecated but keep adding the information as text logs
// for debugging purposes.
for (auto const& type : default_effect_types_) {
SendLogMessage("%s => (Media.Audio.Capture.Win.DefaultEffectType=%s)",
__func__, EffectTypeToString(type));
}
for (auto const& type : raw_effect_types_) {
SendLogMessage("%s => (Media.Audio.Capture.Win.RawEffectType=%s)",
__func__, EffectTypeToString(type));
}
}
if (converter_)
converter_->RemoveInput(this);
ReportAndResetGlitchStats();
// Inform the audio manager that we have been closed. This will cause our
// destruction.
manager_->ReleaseInputStream(this);
}
double WASAPIAudioInputStream::GetMaxVolume() {
// Verify that Open() has been called successfully, to ensure that an audio
// session exists and that an ISimpleAudioVolume interface has been created.
DLOG_IF(ERROR, !opened_) << "Open() has not been called successfully";
if (!opened_)
return 0.0;
// The effective volume value is always in the range 0.0 to 1.0, hence
// we can return a fixed value (=1.0) here.
return 1.0;
}
void WASAPIAudioInputStream::SetVolume(double volume) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK_GE(volume, 0.0);
DCHECK_LE(volume, 1.0);
SendLogMessage("%s({volume=%.2f} [opened=%s])", __func__, volume,
opened_ ? "true" : "false");
if (!opened_)
return;
// Set a new master volume level. Valid volume levels are in the range
// 0.0 to 1.0. Ignore volume-change events.
HRESULT hr = simple_audio_volume_->SetMasterVolume(static_cast<float>(volume),
nullptr);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: ISimpleAudioVolume::SetMasterVolume=[%s])",
__func__, ErrorToString(hr).c_str());
}
// Update the AGC volume level based on the last setting above. Note that,
// the volume-level resolution is not infinite and it is therefore not
// possible to assume that the volume provided as input parameter can be
// used directly. Instead, a new query to the audio hardware is required.
// This method does nothing if AGC is disabled.
UpdateAgcVolume();
}
double WASAPIAudioInputStream::GetVolume() {
DCHECK(opened_) << "Open() has not been called successfully";
if (!opened_)
return 0.0;
// Retrieve the current volume level. The value is in the range 0.0 to 1.0.
float level = 0.0f;
HRESULT hr = simple_audio_volume_->GetMasterVolume(&level);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: ISimpleAudioVolume::GetMasterVolume=[%s])",
__func__, ErrorToString(hr).c_str());
}
return static_cast<double>(level);
}
bool WASAPIAudioInputStream::IsMuted() {
DCHECK(opened_) << "Open() has not been called successfully";
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!opened_)
return false;
// Retrieves the current muting state for the audio session.
BOOL is_muted = FALSE;
HRESULT hr = simple_audio_volume_->GetMute(&is_muted);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: ISimpleAudioVolume::GetMute=[%s])", __func__,
ErrorToString(hr).c_str());
}
return is_muted != FALSE;
}
void WASAPIAudioInputStream::SetOutputDeviceForAec(
const std::string& output_device_id) {
// Not supported. Do nothing.
}
void WASAPIAudioInputStream::SendLogMessage(const char* format, ...) {
if (log_callback_.is_null())
return;
va_list args;
va_start(args, format);
std::string msg("WAIS::" + base::StringPrintV(format, args));
log_callback_.Run(msg);
va_end(args);
}
void WASAPIAudioInputStream::Run() {
ScopedCOMInitializer com_init(ScopedCOMInitializer::kMTA);
// Enable MMCSS to ensure that this thread receives prioritized access to
// CPU resources.
DWORD task_index = 0;
HANDLE mm_task =
avrt::AvSetMmThreadCharacteristics(L"Pro Audio", &task_index);
bool mmcss_is_ok =
(mm_task && avrt::AvSetMmThreadPriority(mm_task, AVRT_PRIORITY_CRITICAL));
if (!mmcss_is_ok) {
// Failed to enable MMCSS on this thread. It is not fatal but can lead
// to reduced QoS at high load.
DWORD err = GetLastError();
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: Failed to enable MMCSS (error code=" << err
<< "))";
}
// Allocate a buffer with a size that enables us to take care of cases like:
// 1) The recorded buffer size is smaller, or does not match exactly with,
// the selected packet size used in each callback.
// 2) The selected buffer size is larger than the recorded buffer size in
// each event.
// In the case where no resampling is required, a single buffer should be
// enough but in case we get buffers that don't match exactly, we'll go with
// two. Same applies if we need to resample and the buffer ratio is perfect.
// However if the buffer ratio is imperfect, we will need 3 buffers to safely
// be able to buffer up data in cases where a conversion requires two audio
// buffers (and we need to be able to write to the third one).
size_t capture_buffer_size =
std::max(2 * endpoint_buffer_size_frames_ * frame_size_bytes_,
2 * packet_size_frames_ * frame_size_bytes_);
int buffers_required = capture_buffer_size / packet_size_bytes_;
if (converter_ && imperfect_buffer_size_conversion_)
++buffers_required;
DCHECK(!fifo_);
fifo_ = std::make_unique<AudioBlockFifo>(
input_format_.Format.nChannels, packet_size_frames_, buffers_required);
DVLOG(1) << "AudioBlockFifo buffer count: " << buffers_required;
bool recording = true;
bool error = false;
HANDLE wait_array[2] = {stop_capture_event_.Get(),
audio_samples_ready_event_.Get()};
record_start_time_ = base::TimeTicks::Now();
last_capture_time_ = base::TimeTicks();
max_timestamp_diff_ = base::TimeDelta::Min();
min_timestamp_diff_ = base::TimeDelta::Max();
while (recording && !error) {
// Wait for a close-down event or a new capture event.
DWORD wait_result = WaitForMultipleObjects(2, wait_array, FALSE, INFINITE);
switch (wait_result) {
case WAIT_OBJECT_0 + 0:
// |stop_capture_event_| has been set.
recording = false;
break;
case WAIT_OBJECT_0 + 1:
// |audio_samples_ready_event_| has been set.
PullCaptureDataAndPushToSink();
break;
case WAIT_FAILED:
default:
error = true;
break;
}
}
if (recording && error) {
// TODO(henrika): perhaps it worth improving the cleanup here by e.g.
// stopping the audio client, joining the thread etc.?
NOTREACHED() << "WASAPI capturing failed with error code "
<< GetLastError();
}
// Disable MMCSS.
if (mm_task && !avrt::AvRevertMmThreadCharacteristics(mm_task)) {
PLOG(WARNING) << "Failed to disable MMCSS";
}
fifo_.reset();
}
void WASAPIAudioInputStream::PullCaptureDataAndPushToSink() {
TRACE_EVENT1("audio", "WASAPIAudioInputStream::PullCaptureDataAndPushToSink",
"sample rate", input_format_.Format.nSamplesPerSec);
UINT64 last_device_position = 0;
UINT32 num_frames_in_next_packet = 0;
// Get the number of frames in the next data packet in the capture endpoint
// buffer. The count reported by GetNextPacketSize matches the count retrieved
// in the GetBuffer call that follows this call.
HRESULT hr =
audio_capture_client_->GetNextPacketSize(&num_frames_in_next_packet);
if (FAILED(hr)) {
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: 1-IAudioCaptureClient::GetNextPacketSize=["
<< ErrorToString(hr).c_str() << "])";
return;
}
// Pull data from the capture endpoint buffer until it's empty or an error
// occurs. Drains the WASAPI capture buffer fully.
while (num_frames_in_next_packet > 0) {
BYTE* data_ptr = nullptr;
UINT32 num_frames_to_read = 0;
DWORD flags = 0;
UINT64 device_position = 0;
UINT64 capture_time_100ns = 0;
// Retrieve the amount of data in the capture endpoint buffer, replace it
// with silence if required, create callbacks for each packet and store
// non-delivered data for the next event.
hr =
audio_capture_client_->GetBuffer(&data_ptr, &num_frames_to_read, &flags,
&device_position, &capture_time_100ns);
if (hr == AUDCLNT_S_BUFFER_EMPTY) {
DCHECK_EQ(num_frames_to_read, 0u);
return;
}
if (hr == AUDCLNT_E_OUT_OF_ORDER) {
// A previous IAudioCaptureClient::GetBuffer() call is still in effect.
// Release any acquired buffer to be able to try reading a buffer again.
audio_capture_client_->ReleaseBuffer(num_frames_to_read);
}
if (FAILED(hr)) {
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: IAudioCaptureClient::GetBuffer=["
<< ErrorToString(hr).c_str() << "])";
return;
}
// The data in the packet is not correlated with the previous packet's
// device position; this is possibly due to a stream state transition or
// timing glitch. Note that, usage of this flag was added after the existing
// glitch detection and it will be used as a supplementary scheme initially.
// The behavior of the AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY flag is
// undefined on the application's first call to GetBuffer after Start and
// Windows 7 or later is required for support.
const bool observed_data_discontinuity =
(device_position > 0 && flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY);
if (observed_data_discontinuity) {
LOG(WARNING) << "WAIS::" << __func__
<< " => (WARNING: AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY)";
}
data_discontinuity_reporter_->Log(observed_data_discontinuity);
// The time at which the device's stream position was recorded is uncertain.
// Thus, the client might be unable to accurately set a time stamp for the
// current data packet.
bool timestamp_error_was_detected = false;
if (flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) {
// TODO(https://crbug.com/825744): it might be possible to improve error
// handling here and avoid using the counter in |capture_time_100ns|.
LOG(WARNING) << "WAIS::" << __func__
<< " => (WARNING: AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR)";
if (num_timestamp_errors_ == 0) {
// Measure the time it took until the first timestamp error was found.
time_until_first_timestamp_error_ =
base::TimeTicks::Now() - record_start_time_;
}
++num_timestamp_errors_;
timestamp_error_was_detected = true;
}
// If the device position has changed, we assume this data belongs to a new
// chunk, so we report delay and glitch stats and update the last and next
// expected device positions.
// If the device position has not changed we assume this data belongs to the
// previous chunk, and only update the expected next device position.
if (device_position != last_device_position) {
if (expected_next_device_position_ != 0) {
base::TimeDelta glitch_duration;
if (device_position > expected_next_device_position_) {
glitch_duration = AudioTimestampHelper::FramesToTime(
device_position - expected_next_device_position_,
input_format_.Format.nSamplesPerSec);
}
glitch_reporter_.UpdateStats(glitch_duration);
}
last_device_position = device_position;
expected_next_device_position_ = device_position + num_frames_to_read;
} else {
expected_next_device_position_ += num_frames_to_read;
}
base::TimeTicks capture_time;
if (use_fake_audio_capture_timestamps_) {
capture_time = base::TimeTicks::Now();
} else if (!timestamp_error_was_detected) {
// Use the latest |capture_time_100ns| since it is marked as valid.
capture_time += base::Microseconds(capture_time_100ns / 10.0);
}
if (capture_time <= last_capture_time_) {
// Latest |capture_time_100ns| can't be trusted. Ensure a monotonic time-
// stamp sequence by adding one microsecond to the latest timestamp.
capture_time = last_capture_time_ + base::Microseconds(1);
}
// Keep track of max and min time difference between two successive time-
// stamps. Results are used in Stop() to verify that the time-stamp sequence
// was monotonic.
if (!last_capture_time_.is_null()) {
const auto delta_ts = capture_time - last_capture_time_;
DCHECK_GT(device_position, 0u);
DCHECK_GT(delta_ts, base::TimeDelta::Min());
if (delta_ts > max_timestamp_diff_) {
max_timestamp_diff_ = delta_ts;
} else if (delta_ts < min_timestamp_diff_) {
min_timestamp_diff_ = delta_ts;
}
}
// Store the capture timestamp. Might be used as reference next time if
// a new valid timestamp can't be retrieved to always guarantee a monotonic
// sequence.
last_capture_time_ = capture_time;
// Adjust |capture_time| for the FIFO before pushing.
capture_time -= AudioTimestampHelper::FramesToTime(
fifo_->GetAvailableFrames(), input_format_.Format.nSamplesPerSec);
if (flags & AUDCLNT_BUFFERFLAGS_SILENT) {
fifo_->PushSilence(num_frames_to_read);
} else {
const int bytes_per_sample = input_format_.Format.wBitsPerSample / 8;
peak_detector_.FindPeak(data_ptr, num_frames_to_read, bytes_per_sample);
fifo_->Push(data_ptr, num_frames_to_read, bytes_per_sample);
}
hr = audio_capture_client_->ReleaseBuffer(num_frames_to_read);
if (FAILED(hr)) {
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: IAudioCaptureClient::ReleaseBuffer=["
<< ErrorToString(hr).c_str() << "])";
return;
}
// Get a cached AGC volume level which is updated once every second on the
// audio manager thread. Note that, |volume| is also updated each time
// SetVolume() is called through IPC by the render-side AGC.
double volume = 0.0;
GetAgcVolume(&volume);
// Deliver captured data to the registered consumer using a packet size
// which was specified at construction.
while (fifo_->available_blocks()) {
if (converter_) {
if (imperfect_buffer_size_conversion_ &&
fifo_->available_blocks() == 1) {
// Special case. We need to buffer up more audio before we can convert
// or else we'll suffer an underrun.
// TODO(grunell): Verify this is really true.
return;
}
converter_->Convert(convert_bus_.get());
sink_->OnData(convert_bus_.get(), capture_time, volume, {});
// Move the capture time forward for each vended block.
capture_time += AudioTimestampHelper::FramesToTime(
convert_bus_->frames(), output_format_.nSamplesPerSec);
} else {