-
Notifications
You must be signed in to change notification settings - Fork 6k
/
SilenceSkippingAudioProcessor.java
376 lines (341 loc) · 13.1 KB
/
SilenceSkippingAudioProcessor.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.audio;
import static java.lang.Math.min;
import static java.lang.annotation.ElementType.TYPE_USE;
import androidx.annotation.IntDef;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.Util;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.nio.ByteBuffer;
/**
* An {@link AudioProcessor} that skips silence in the input stream. Input and output are 16-bit
* PCM.
*/
public final class SilenceSkippingAudioProcessor extends BaseAudioProcessor {
/**
* The default value for {@link #SilenceSkippingAudioProcessor(long, long, short)
* minimumSilenceDurationUs}.
*/
public static final long DEFAULT_MINIMUM_SILENCE_DURATION_US = 150_000;
/**
* The default value for {@link #SilenceSkippingAudioProcessor(long, long, short)
* paddingSilenceUs}.
*/
public static final long DEFAULT_PADDING_SILENCE_US = 20_000;
/**
* The default value for {@link #SilenceSkippingAudioProcessor(long, long, short)
* silenceThresholdLevel}.
*/
public static final short DEFAULT_SILENCE_THRESHOLD_LEVEL = 1024;
/** Trimming states. */
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({
STATE_NOISY,
STATE_MAYBE_SILENT,
STATE_SILENT,
})
private @interface State {}
/** State when the input is not silent. */
private static final int STATE_NOISY = 0;
/** State when the input may be silent but we haven't read enough yet to know. */
private static final int STATE_MAYBE_SILENT = 1;
/** State when the input is silent. */
private static final int STATE_SILENT = 2;
private final long minimumSilenceDurationUs;
private final long paddingSilenceUs;
private final short silenceThresholdLevel;
private int bytesPerFrame;
private boolean enabled;
/**
* Buffers audio data that may be classified as silence while in {@link #STATE_MAYBE_SILENT}. If
* the input becomes noisy before the buffer has filled, it will be output. Otherwise, the buffer
* contents will be dropped and the state will transition to {@link #STATE_SILENT}.
*/
private byte[] maybeSilenceBuffer;
/**
* Stores the latest part of the input while silent. It will be output as padding if the next
* input is noisy.
*/
private byte[] paddingBuffer;
private @State int state;
private int maybeSilenceBufferSize;
private int paddingSize;
private boolean hasOutputNoise;
private long skippedFrames;
/** Creates a new silence skipping audio processor. */
public SilenceSkippingAudioProcessor() {
this(
DEFAULT_MINIMUM_SILENCE_DURATION_US,
DEFAULT_PADDING_SILENCE_US,
DEFAULT_SILENCE_THRESHOLD_LEVEL);
}
/**
* Creates a new silence skipping audio processor.
*
* @param minimumSilenceDurationUs The minimum duration of audio that must be below {@code
* silenceThresholdLevel} to classify that part of audio as silent, in microseconds.
* @param paddingSilenceUs The duration of silence by which to extend non-silent sections, in
* microseconds. The value must not exceed {@code minimumSilenceDurationUs}.
* @param silenceThresholdLevel The absolute level below which an individual PCM sample is
* classified as silent.
*/
public SilenceSkippingAudioProcessor(
long minimumSilenceDurationUs, long paddingSilenceUs, short silenceThresholdLevel) {
Assertions.checkArgument(paddingSilenceUs <= minimumSilenceDurationUs);
this.minimumSilenceDurationUs = minimumSilenceDurationUs;
this.paddingSilenceUs = paddingSilenceUs;
this.silenceThresholdLevel = silenceThresholdLevel;
maybeSilenceBuffer = Util.EMPTY_BYTE_ARRAY;
paddingBuffer = Util.EMPTY_BYTE_ARRAY;
}
/**
* Sets whether to skip silence in the input. This method may only be called after draining data
* through the processor. The value returned by {@link #isActive()} may change, and the processor
* must be {@link #flush() flushed} before queueing more data.
*
* @param enabled Whether to skip silence in the input.
*/
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
/**
* Returns the total number of frames of input audio that were skipped due to being classified as
* silence since the last call to {@link #flush()}.
*/
public long getSkippedFrames() {
return skippedFrames;
}
// AudioProcessor implementation.
@Override
public AudioFormat onConfigure(AudioFormat inputAudioFormat)
throws UnhandledAudioFormatException {
if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT) {
throw new UnhandledAudioFormatException(inputAudioFormat);
}
return enabled ? inputAudioFormat : AudioFormat.NOT_SET;
}
@Override
public boolean isActive() {
return enabled;
}
@Override
public void queueInput(ByteBuffer inputBuffer) {
while (inputBuffer.hasRemaining() && !hasPendingOutput()) {
switch (state) {
case STATE_NOISY:
processNoisy(inputBuffer);
break;
case STATE_MAYBE_SILENT:
processMaybeSilence(inputBuffer);
break;
case STATE_SILENT:
processSilence(inputBuffer);
break;
default:
throw new IllegalStateException();
}
}
}
@Override
protected void onQueueEndOfStream() {
if (maybeSilenceBufferSize > 0) {
// We haven't received enough silence to transition to the silent state, so output the buffer.
output(maybeSilenceBuffer, maybeSilenceBufferSize);
}
if (!hasOutputNoise) {
skippedFrames += paddingSize / bytesPerFrame;
}
}
@Override
protected void onFlush() {
if (enabled) {
bytesPerFrame = inputAudioFormat.bytesPerFrame;
int maybeSilenceBufferSize = durationUsToFrames(minimumSilenceDurationUs) * bytesPerFrame;
if (maybeSilenceBuffer.length != maybeSilenceBufferSize) {
maybeSilenceBuffer = new byte[maybeSilenceBufferSize];
}
paddingSize = durationUsToFrames(paddingSilenceUs) * bytesPerFrame;
if (paddingBuffer.length != paddingSize) {
paddingBuffer = new byte[paddingSize];
}
}
state = STATE_NOISY;
skippedFrames = 0;
maybeSilenceBufferSize = 0;
hasOutputNoise = false;
}
@Override
protected void onReset() {
enabled = false;
paddingSize = 0;
maybeSilenceBuffer = Util.EMPTY_BYTE_ARRAY;
paddingBuffer = Util.EMPTY_BYTE_ARRAY;
}
// Internal methods.
/**
* Incrementally processes new input from {@code inputBuffer} while in {@link #STATE_NOISY},
* updating the state if needed.
*/
private void processNoisy(ByteBuffer inputBuffer) {
int limit = inputBuffer.limit();
// Check if there's any noise within the maybe silence buffer duration.
inputBuffer.limit(min(limit, inputBuffer.position() + maybeSilenceBuffer.length));
int noiseLimit = findNoiseLimit(inputBuffer);
if (noiseLimit == inputBuffer.position()) {
// The buffer contains the start of possible silence.
state = STATE_MAYBE_SILENT;
} else {
inputBuffer.limit(noiseLimit);
output(inputBuffer);
}
// Restore the limit.
inputBuffer.limit(limit);
}
/**
* Incrementally processes new input from {@code inputBuffer} while in {@link
* #STATE_MAYBE_SILENT}, updating the state if needed.
*/
private void processMaybeSilence(ByteBuffer inputBuffer) {
int limit = inputBuffer.limit();
int noisePosition = findNoisePosition(inputBuffer);
int maybeSilenceInputSize = noisePosition - inputBuffer.position();
int maybeSilenceBufferRemaining = maybeSilenceBuffer.length - maybeSilenceBufferSize;
if (noisePosition < limit && maybeSilenceInputSize < maybeSilenceBufferRemaining) {
// The maybe silence buffer isn't full, so output it and switch back to the noisy state.
output(maybeSilenceBuffer, maybeSilenceBufferSize);
maybeSilenceBufferSize = 0;
state = STATE_NOISY;
} else {
// Fill as much of the maybe silence buffer as possible.
int bytesToWrite = min(maybeSilenceInputSize, maybeSilenceBufferRemaining);
inputBuffer.limit(inputBuffer.position() + bytesToWrite);
inputBuffer.get(maybeSilenceBuffer, maybeSilenceBufferSize, bytesToWrite);
maybeSilenceBufferSize += bytesToWrite;
if (maybeSilenceBufferSize == maybeSilenceBuffer.length) {
// We've reached a period of silence, so skip it, taking in to account padding for both
// the noisy to silent transition and any future silent to noisy transition.
if (hasOutputNoise) {
output(maybeSilenceBuffer, paddingSize);
skippedFrames += (maybeSilenceBufferSize - paddingSize * 2) / bytesPerFrame;
} else {
skippedFrames += (maybeSilenceBufferSize - paddingSize) / bytesPerFrame;
}
updatePaddingBuffer(inputBuffer, maybeSilenceBuffer, maybeSilenceBufferSize);
maybeSilenceBufferSize = 0;
state = STATE_SILENT;
}
// Restore the limit.
inputBuffer.limit(limit);
}
}
/**
* Incrementally processes new input from {@code inputBuffer} while in {@link #STATE_SILENT},
* updating the state if needed.
*/
private void processSilence(ByteBuffer inputBuffer) {
int limit = inputBuffer.limit();
int noisyPosition = findNoisePosition(inputBuffer);
inputBuffer.limit(noisyPosition);
skippedFrames += inputBuffer.remaining() / bytesPerFrame;
updatePaddingBuffer(inputBuffer, paddingBuffer, paddingSize);
if (noisyPosition < limit) {
// Output the padding, which may include previous input as well as new input, then transition
// back to the noisy state.
output(paddingBuffer, paddingSize);
state = STATE_NOISY;
// Restore the limit.
inputBuffer.limit(limit);
}
}
/**
* Copies {@code length} elements from {@code data} to populate a new output buffer from the
* processor.
*/
private void output(byte[] data, int length) {
replaceOutputBuffer(length).put(data, 0, length).flip();
if (length > 0) {
hasOutputNoise = true;
}
}
/**
* Copies remaining bytes from {@code data} to populate a new output buffer from the processor.
*/
private void output(ByteBuffer data) {
int length = data.remaining();
replaceOutputBuffer(length).put(data).flip();
if (length > 0) {
hasOutputNoise = true;
}
}
/**
* Fills {@link #paddingBuffer} using data from {@code input}, plus any additional buffered data
* at the end of {@code buffer} (up to its {@code size}) required to fill it, advancing the input
* position.
*/
private void updatePaddingBuffer(ByteBuffer input, byte[] buffer, int size) {
int fromInputSize = min(input.remaining(), paddingSize);
int fromBufferSize = paddingSize - fromInputSize;
System.arraycopy(
/* src= */ buffer,
/* srcPos= */ size - fromBufferSize,
/* dest= */ paddingBuffer,
/* destPos= */ 0,
/* length= */ fromBufferSize);
input.position(input.limit() - fromInputSize);
input.get(paddingBuffer, fromBufferSize, fromInputSize);
}
/**
* Returns the number of input frames corresponding to {@code durationUs} microseconds of audio.
*/
private int durationUsToFrames(long durationUs) {
return (int) ((durationUs * inputAudioFormat.sampleRate) / C.MICROS_PER_SECOND);
}
/**
* Returns the earliest byte position in [position, limit) of {@code buffer} that contains a frame
* classified as a noisy frame, or the limit of the buffer if no such frame exists.
*/
private int findNoisePosition(ByteBuffer buffer) {
// The input is in ByteOrder.nativeOrder(), which is little endian on Android.
for (int i = buffer.position(); i < buffer.limit(); i += 2) {
if (Math.abs(buffer.getShort(i)) > silenceThresholdLevel) {
// Round to the start of the frame.
return bytesPerFrame * (i / bytesPerFrame);
}
}
return buffer.limit();
}
/**
* Returns the earliest byte position in [position, limit) of {@code buffer} such that all frames
* from the byte position to the limit are classified as silent.
*/
private int findNoiseLimit(ByteBuffer buffer) {
// The input is in ByteOrder.nativeOrder(), which is little endian on Android.
for (int i = buffer.limit() - 2; i >= buffer.position(); i -= 2) {
if (Math.abs(buffer.getShort(i)) > silenceThresholdLevel) {
// Return the start of the next frame.
return bytesPerFrame * (i / bytesPerFrame) + bytesPerFrame;
}
}
return buffer.position();
}
}