Skip to content

Commit

Permalink
Add initial support for configurable codecs
Browse files Browse the repository at this point in the history 
This commit adds the initial framework for allowing the configuration of
output codecs. There are three codec choices available:

* FLAC: (default) lossless, compression level 0-8
* OGG/Opus: lossy, bitrate 6-510kbps, Android 10+ only
* M4A/AAC: lossy, HE-AAC bitrate 24-32kbps, AAC-LC bitrate 32-128kbps

While all three codecs are implemented, RecorderThread is currently
hardcoded to use the default. The next step is to save and load the
codec configuration from SharedPreferences and the finally add the user
interface.

Issue: #21
Signed-off-by: Andrew Gunnerson <chillermillerlong@hotmail.com>
  • Loading branch information
@chenxiaolong
chenxiaolong committed May 25, 2022
1 parent 7320c41 commit 3d3eb82
Show file tree
Hide file tree
Showing 9 changed files with 484 additions and 142 deletions.
197 changes: 55 additions & 142 deletions app/src/main/java/com/chiller3/bcr/RecorderThread.kt
View file
@@ -1,24 +1,22 @@
@file:Suppress("OPT_IN_IS_NOT_ENABLED")
@file:OptIn(ExperimentalUnsignedTypes::class)

package com.chiller3.bcr

import android.annotation.SuppressLint
import android.content.Context
import android.media.*
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaCodec
import android.media.MediaRecorder
import android.net.Uri
import android.os.Build
import android.os.ParcelFileDescriptor
import android.system.Os
import android.system.OsConstants
import android.telecom.Call
import android.telecom.PhoneAccount
import android.util.Log
import androidx.documentfile.provider.DocumentFile
import java.io.FileDescriptor
import com.chiller3.bcr.codec.Codec
import com.chiller3.bcr.codec.Container
import java.io.IOException
import java.lang.Integer.min
import java.nio.ByteBuffer
import java.time.Instant
import java.time.ZoneId
import java.time.ZonedDateTime
Expand Down Expand Up @@ -60,6 +58,9 @@ class RecorderThread(
}
private val displayName: String? = call.details.callerDisplayName

// TODO
private val codec = Codec.default

init {
Log.i(TAG, "[${id}] Created thread for call: $call")
}
Expand Down Expand Up @@ -136,8 +137,8 @@ class RecorderThread(
data class OutputFile(val uri: Uri, val pfd: ParcelFileDescriptor)

/**
* Try to create and open a new FLAC file in the user-chosen directory if possible and fall back
* to the default output directory if not. [name] should not contain a file extension.
* Try to create and open a new output file in the user-chosen directory if possible and fall
* back to the default output directory if not. [name] should not contain a file extension.
*
* @throws IOException if the file could not be created in either directory
*/
Expand All @@ -160,13 +161,13 @@ class RecorderThread(
}

/**
* Create and open a new FLAC file with name [name] inside [directory]. [name] should not
* contain a file extension.
* Create and open a new output file with name [name] inside [directory]. [name] should not
* contain a file extension. The file extension is automatically determined from [codec].
*
* @throws IOException if file creation or opening fails
*/
private fun openOutputFileInDir(directory: DocumentFile, name: String): OutputFile {
val file = directory.createFile(MediaFormat.MIMETYPE_AUDIO_FLAC, name)
val file = directory.createFile(codec.mimeTypeContainer, name)
?: throw IOException("Failed to create file in ${directory.uri}")
val pfd = context.contentResolver.openFileDescriptor(file.uri, "rw")
?: throw IOException("Failed to open file at ${file.uri}")
Expand Down Expand Up @@ -197,19 +198,26 @@ class RecorderThread(
audioRecord.startRecording()

try {
val codec = getFlacCodec(audioFormat, audioRecord.sampleRate)
val mediaFormat = codec.getMediaFormat(audioFormat, audioRecord.sampleRate)
val mediaCodec = codec.getMediaCodec(mediaFormat)

try {
codec.start()
mediaCodec.start()

try {
val frames = encodeLoop(audioRecord, codec, pfd.fileDescriptor)
setFlacHeaderDuration(pfd.fileDescriptor, frames)
val container = codec.getContainer(pfd.fileDescriptor)

try {
encodeLoop(audioRecord, mediaCodec, container)
container.stop()
} finally {
container.release()
}
} finally {
codec.stop()
mediaCodec.stop()
}
} finally {
codec.release()
mediaCodec.release()
}
} finally {
audioRecord.stop()
Expand All @@ -223,40 +231,36 @@ class RecorderThread(
* Main loop for encoding captured raw audio into an output file.
*
* The loop runs forever until [cancel] is called. At that point, no further data will be read
* from [audioRecord] and the remaining output data from [codec] will be written to [fd].
* If [audioRecord] fails to capture data, the loop will behave as if [cancel] was called
* (ie. abort, but ensuring that the output file is valid).
* from [audioRecord] and the remaining output data from [mediaCodec] will be written to
* [container]. If [audioRecord] fails to capture data, the loop will behave as if [cancel] was
* called (ie. abort, but ensuring that the output file is valid).
*
* The approximate amount of time to cancel reading from the audio source is 100ms. This does
* not include the time required to write out the remaining encoded data to the output file.
*
* @param audioRecord [AudioRecord.startRecording] must have been called
* @param codec [MediaCodec.start] must have been called
* @param fd Will be truncated to 0 bytes before writing
* @param mediaCodec [MediaCodec.start] must have been called
* @param container [Container.start] must *not* have been called. It will be left in a started
* state after this method returns.
*
* @throws MediaCodec.CodecException if the codec encounters an error
*
* @return The number of frames that were recorded
*/
private fun encodeLoop(audioRecord: AudioRecord, codec: MediaCodec, fd: FileDescriptor): ULong {
Os.lseek(fd, 0, OsConstants.SEEK_SET)
Os.ftruncate(fd, 0)

private fun encodeLoop(audioRecord: AudioRecord, mediaCodec: MediaCodec, container: Container) {
// This is the most we ever read from audioRecord, even if the codec input buffer is
// larger. This is purely for fast'ish cancellation and not for latency.
val maxSamplesInBytes = audioRecord.sampleRate / 10 * getFrameSize(audioRecord.format)

val inputTimestamp = 0L
var inputTimestamp = 0L
var inputComplete = false
val bufferInfo = MediaCodec.BufferInfo()
val frameSize = getFrameSize(audioRecord.format)
var totalFrames = 0uL
var trackIndex = -1

while (true) {
if (!inputComplete) {
val inputBufferId = codec.dequeueInputBuffer(10000)
val inputBufferId = mediaCodec.dequeueInputBuffer(10000)
if (inputBufferId >= 0) {
val buffer = codec.getInputBuffer(inputBufferId)!!
val buffer = mediaCodec.getInputBuffer(inputBufferId)!!

val maxRead = min(maxSamplesInBytes, buffer.remaining())
val n = audioRecord.read(buffer, maxRead)
Expand All @@ -270,16 +274,13 @@ class RecorderThread(
// behavior
Log.e(TAG, "MediaCodec's ByteBuffer was not a direct buffer")
isCancelled = true
} else {
val frames = n / frameSize
inputTimestamp += frames * 1_000_000L / audioRecord.sampleRate
}

val frames = n / frameSize

// Setting the presentation timestamp will cause `c2.android.flac.encoder`
// software encoder to crash with SIGABRT
//inputTimestamp += frames * 1000000 / audioRecord.sampleRate

if (isCancelled) {
val duration = "%.1f".format(inputTimestamp / 1000000f)
val duration = "%.1f".format(inputTimestamp / 1_000_000.0)
Log.d(TAG, "Input complete after ${duration}s")
inputComplete = true
}
Expand All @@ -290,42 +291,42 @@ class RecorderThread(
0
}

codec.queueInputBuffer(inputBufferId, 0, n, inputTimestamp, flags)

totalFrames += frames.toULong()
// Setting the presentation timestamp will cause `c2.android.flac.encoder`
// software encoder to crash with SIGABRT
mediaCodec.queueInputBuffer(inputBufferId, 0, n, 0, flags)
} else if (inputBufferId != MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.w(TAG, "Unexpected input buffer dequeue error: $inputBufferId")
}
}

val outputBufferId = codec.dequeueOutputBuffer(bufferInfo, 0)
val outputBufferId = mediaCodec.dequeueOutputBuffer(bufferInfo, 0)
if (outputBufferId >= 0) {
val buffer = codec.getOutputBuffer(outputBufferId)!!
val buffer = mediaCodec.getOutputBuffer(outputBufferId)!!

Os.write(fd, buffer)
container.writeSamples(trackIndex, buffer, bufferInfo)

codec.releaseOutputBuffer(outputBufferId, false)
mediaCodec.releaseOutputBuffer(outputBufferId, false)

if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
// Output has been fully written
break
}
} else if (outputBufferId != MediaCodec.INFO_OUTPUT_FORMAT_CHANGED &&
outputBufferId != MediaCodec.INFO_TRY_AGAIN_LATER) {
} else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
val outputFormat = mediaCodec.outputFormat
Log.d(TAG, "Output format changed to: $outputFormat")
trackIndex = container.addTrack(outputFormat)
container.start()
} else if (outputBufferId != MediaCodec.INFO_TRY_AGAIN_LATER) {
Log.w(TAG, "Unexpected output buffer dequeue error: $outputBufferId")
}
}

return totalFrames
}

companion object {
private val TAG = RecorderThread::class.java.simpleName
private const val CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO
private const val ENCODING = AudioFormat.ENCODING_PCM_16BIT

private val FLAC_MAGIC = ubyteArrayOf(0x66u, 0x4cu, 0x61u, 0x43u) // fLaC

// Eg. 20220429_180249.123-0400
private val FORMATTER = DateTimeFormatterBuilder()
.appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD)
Expand All @@ -339,41 +340,6 @@ class RecorderThread(
.appendOffset("+HHMMss", "+0000")
.toFormatter()

/**
* Create a [MediaCodec] encoder for FLAC using the given PCM audio format as the input.
*
* @throws Exception if the device does not support encoding FLAC with properties matching
* matching the raw PCM data or if configuring the [MediaCodec] fails.
*/
private fun getFlacCodec(audioFormat: AudioFormat, sampleRate: Int): MediaCodec {
// AOSP ignores this because FLAC compression is lossless, but just in case the system
// uses another FLAC encoder that requiress a non-dummy value (eg. 0), we'll just use
// the PCM s16le bitrate. It's an overestimation, but shouldn't cause any issues.
val bitRate = getFrameSize(audioFormat) * sampleRate / 8

val format = MediaFormat()
format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_FLAC)
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate)
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, audioFormat.channelCount)
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, sampleRate)
format.setInteger(MediaFormat.KEY_FLAC_COMPRESSION_LEVEL, 8)

val encoder = MediaCodecList(MediaCodecList.REGULAR_CODECS).findEncoderForFormat(format)
?: throw Exception("No FLAC encoder found")
Log.d(TAG, "FLAC encoder: $encoder")

val codec = MediaCodec.createByCodecName(encoder)

try {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
} catch (e: Exception) {
codec.release()
throw e
}

return codec
}

private fun getFrameSize(audioFormat: AudioFormat): Int {
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
audioFormat.frameSizeInBytes
Expand All @@ -383,59 +349,6 @@ class RecorderThread(
2 * audioFormat.channelCount
}
}

/**
* Write the frame count [frames] to the STREAMINFO metadata block of a flac file. [frames]
* should not account for the number of channels (ie. frames, not samples).
*
* @throws IOException If FLAC metadata does not appear to be valid
* @throws IllegalArgumentException if [frames] exceeds the bounds of a 36-bit integer
*/
private fun setFlacHeaderDuration(fd: FileDescriptor, frames: ULong) {
if (frames >= 2uL.shl(36)) {
throw IllegalArgumentException("Frame count cannot be represented in FLAC: $frames")
}

Os.lseek(fd, 0, OsConstants.SEEK_SET)

// Magic (4 bytes)
// + metadata block header (4 bytes)
// + streaminfo block (34 bytes)
val buf = UByteArray(42)

if (Os.read(fd, buf.asByteArray(), 0, buf.size) != buf.size) {
throw IOException("EOF reached when reading FLAC headers")
}

// Validate the magic
if (ByteBuffer.wrap(buf.asByteArray(), 0, 4) !=
ByteBuffer.wrap(FLAC_MAGIC.asByteArray())) {
throw IOException("FLAC magic not found")
}

// Validate that the first metadata block is STREAMINFO and has the correct size
if (buf[4] and 0x7fu != 0.toUByte()) {
throw IOException("First metadata block is not STREAMINFO")
}

val streamInfoSize = buf[5].toUInt().shl(16) or
buf[6].toUInt().shl(8) or buf[7].toUInt()
if (streamInfoSize < 34u) {
throw IOException("STREAMINFO block is too small")
}

// Total samples field is a 36-bit integer that begins 4 bits into the 21st byte
buf[21] = (buf[21] and 0xf0u) or (frames.shr(32) and 0xfu).toUByte()
buf[22] = (frames.shr(24) and 0xffu).toUByte()
buf[23] = (frames.shr(16) and 0xffu).toUByte()
buf[24] = (frames.shr(8) and 0xffu).toUByte()
buf[25] = (frames and 0xffu).toUByte()

Os.lseek(fd, 21, OsConstants.SEEK_SET)
if (Os.write(fd, buf.asByteArray(), 21, 5) != 5) {
throw IOException("EOF reached when writing frame count")
}
}
}

interface OnRecordingCompletedListener {
Expand Down
36 changes: 36 additions & 0 deletions app/src/main/java/com/chiller3/bcr/codec/AacCodec.kt
View file
@@ -0,0 +1,36 @@
package com.chiller3.bcr.codec

import android.media.AudioFormat
import android.media.MediaCodecInfo
import android.media.MediaFormat
import android.media.MediaMuxer
import java.io.FileDescriptor

class AacCodec : Codec() {
override val codecParamType: CodecParamType = CodecParamType.Bitrate
// The codec has no hard limits, so the lower bound is ffmpeg's recommended minimum bitrate for
// HE-AAC: 24kbps/channel. The upper bound is twice the bitrate for audible transparency with
// AAC-LC: 2 * 64kbps/channel.
// https://trac.ffmpeg.org/wiki/Encode/AAC
override val codecParamRange: UIntRange = 24_000u..128_000u
override val codecParamDefault: UInt = 64_000u
// https://datatracker.ietf.org/doc/html/rfc6381#section-3.1
override val mimeTypeContainer: String = "audio/mp4"
override val mimeTypeAudio: String = MediaFormat.MIMETYPE_AUDIO_AAC
override val supported: Boolean = true

override fun getMediaFormat(audioFormat: AudioFormat, sampleRate: Int): MediaFormat =
super.getMediaFormat(audioFormat, sampleRate).apply {
val profile = if (codecParamValue >= 32_000u) {
MediaCodecInfo.CodecProfileLevel.AACObjectLC
} else {
MediaCodecInfo.CodecProfileLevel.AACObjectHE
}

setInteger(MediaFormat.KEY_AAC_PROFILE, profile)
setInteger(MediaFormat.KEY_BIT_RATE, codecParamValue.toInt() * audioFormat.channelCount)
}

override fun getContainer(fd: FileDescriptor): Container =
MediaMuxerContainer(fd, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
}

0 comments on commit 3d3eb82

Please sign in to comment.