classify audio attachments (#29197)

* classify audio attachments

* compute amps

* better audio attachment support

* x

* x

Author: zoom-ua

go/bind/keybase.go

@@ -5,9 +5,11 @@ package keybase
 
 import (
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math"
 	"net"
 	"os"
 	"path/filepath"
@@ -102,6 +104,9 @@ type PushNotifier interface {
 type NativeVideoHelper interface {
 	Thumbnail(filename string) []byte
 	Duration(filename string) int
+	// AudioAmps returns IEEE 754 float32 samples encoded as little-endian bytes,
+	// representing RMS amplitude values in [0,1] for waveform visualization.
+	AudioAmps(filename string) []byte
 }
 
 // ShareIntentDonator is implemented by the native iOS layer to donate INSendMessageIntent
@@ -189,6 +194,19 @@ func (v videoHelper) ThumbnailAndDuration(ctx context.Context, filename string)
 	return v.nvh.Thumbnail(filename), v.nvh.Duration(filename), nil
 }
 
+func (v videoHelper) AudioAmps(ctx context.Context, filename string) ([]float64, error) {
+	data := v.nvh.AudioAmps(filename)
+	if len(data) == 0 || len(data)%4 != 0 {
+		return nil, nil
+	}
+	amps := make([]float64, len(data)/4)
+	for i := range amps {
+		bits := binary.LittleEndian.Uint32(data[i*4:])
+		amps[i] = float64(math.Float32frombits(bits))
+	}
+	return amps, nil
+}
+
 type ExternalDNSNSFetcher interface {
 	GetServers() []byte
 }

go/chat/attachments/preprocess.go

@@ -278,6 +278,8 @@ func PreprocessAsset(ctx context.Context, g *globals.Context, log utils.DebugLab
 			p.PreviewDim = &Dimension{Width: previewRes.PreviewWidth, Height: previewRes.PreviewHeight}
 		}
 		p.BaseDurationMs = previewRes.BaseDurationMs
+		p.BaseIsAudio = previewRes.BaseIsAudio
+		p.PreviewAudioAmps = previewRes.AudioAmps
 		p.PreviewDurationMs = previewRes.PreviewDurationMs
 	}
 

go/chat/attachments/preview.go

@@ -37,6 +37,8 @@ type PreviewRes struct {
 	BaseWidth         int
 	BaseHeight        int
 	BaseDurationMs    int
+	BaseIsAudio       bool
+	AudioAmps         []float64
 	PreviewWidth      int
 	PreviewHeight     int
 	PreviewDurationMs int

go/chat/attachments/preview_android.go

@@ -21,6 +21,13 @@ func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader,
 		return res, err
 	}
 	log.Debug(ctx, "previewVideo: size: %d duration: %d", len(dat), duration)
+	if len(dat) == 0 && duration > 1 && isAudioExtension(basename) {
+		amps, ampErr := nvh.AudioAmps(ctx, basename)
+		if ampErr != nil {
+			log.Debug(ctx, "previewVideo: AudioAmps failed: %v", ampErr)
+		}
+		return previewAudio(duration, amps)
+	}
 	if len(dat) == 0 {
 		log.Debug(ctx, "failed to generate preview from native, using blank image")
 		return previewVideoBlank(ctx, log, src, basename)

go/chat/attachments/preview_darwin.go

@@ -12,6 +12,7 @@ package attachments
 #include <CoreFoundation/CoreFoundation.h>
 #include <Foundation/Foundation.h>
 #include <ImageIO/ImageIO.h>
+#include <math.h>
 #include <UniformTypeIdentifiers/UniformTypeIdentifiers.h>
 #if TARGET_OS_IPHONE
 #include <MobileCoreServices/MobileCoreServices.h>
@@ -31,6 +32,11 @@ typedef struct {
 	int imageLength;
 } ImageConversionResult;
 
+typedef struct {
+	float* data;
+	int length;
+} AudioAmpsResult;
+
 VideoPreviewResult MakeVideoThumbnail(const char* inFilename) {
 	VideoPreviewResult result = {NULL, 0, 0};
 	NSString* filename = [NSString stringWithUTF8String:inFilename];
@@ -71,6 +77,118 @@ VideoPreviewResult MakeVideoThumbnail(const char* inFilename) {
 	return result;
 }
 
+// GetAudioAmplitudes reads PCM samples from an audio file via AVAssetReader and
+// returns numSamples RMS amplitude values in [0,1]. The caller must free result.data.
+AudioAmpsResult GetAudioAmplitudes(const char* inFilename, int numSamples) {
+	AudioAmpsResult result = {NULL, 0};
+	if (numSamples <= 0) return result;
+
+	NSString* filename = [NSString stringWithUTF8String:inFilename];
+	NSURL* url = [NSURL fileURLWithPath:filename];
+	AVURLAsset* asset = [AVURLAsset URLAssetWithURL:url options:nil];
+
+	NSArray<AVAssetTrack*>* audioTracks = [asset tracksWithMediaType:AVMediaTypeAudio];
+	if (audioTracks.count == 0) return result;
+	AVAssetTrack* audioTrack = audioTracks[0];
+
+	Float64 durationSeconds = CMTimeGetSeconds(asset.duration);
+	if (!(durationSeconds > 0)) return result;
+
+	Float64 sampleRate = 0;
+	for (id formatDescription in audioTrack.formatDescriptions) {
+		CMAudioFormatDescriptionRef desc = (__bridge CMAudioFormatDescriptionRef)formatDescription;
+		const AudioStreamBasicDescription* asbd =
+			CMAudioFormatDescriptionGetStreamBasicDescription(desc);
+		if (asbd && asbd->mSampleRate > 0) {
+			sampleRate = asbd->mSampleRate;
+			break;
+		}
+	}
+	if (sampleRate <= 0) {
+		sampleRate = 44100;
+	}
+	long long totalSamples = llround(durationSeconds * sampleRate);
+	if (totalSamples < numSamples) {
+		totalSamples = numSamples;
+	}
+
+	NSError* error = nil;
+	AVAssetReader* reader = [AVAssetReader assetReaderWithAsset:asset error:&error];
+	if (!reader || error) return result;
+
+	NSDictionary* outputSettings = @{
+		AVFormatIDKey:              @(kAudioFormatLinearPCM),
+		AVLinearPCMBitDepthKey:     @32,
+		AVLinearPCMIsFloatKey:      @YES,
+		AVLinearPCMIsNonInterleaved: @NO,
+		AVNumberOfChannelsKey:      @1,
+	};
+
+	AVAssetReaderTrackOutput* output = [AVAssetReaderTrackOutput
+		assetReaderTrackOutputWithTrack:audioTrack
+		outputSettings:outputSettings];
+	output.alwaysCopiesSampleData = NO;
+
+	if (![reader canAddOutput:output]) return result;
+	[reader addOutput:output];
+	if (![reader startReading]) return result;
+
+	float* sumSq = (float*)calloc(numSamples, sizeof(float));
+	unsigned int* counts = (unsigned int*)calloc(numSamples, sizeof(unsigned int));
+	if (!sumSq || !counts) {
+		free(sumSq);
+		free(counts);
+		return result;
+	}
+
+	long long sampleIndex = 0;
+	while (reader.status == AVAssetReaderStatusReading) {
+		CMSampleBufferRef sampleBuffer = [output copyNextSampleBuffer];
+		if (!sampleBuffer) break;
+		CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
+		if (blockBuffer) {
+			size_t length = CMBlockBufferGetDataLength(blockBuffer);
+			if (length >= sizeof(float)) {
+				float* chunk = (float*)malloc(length);
+				if (chunk && CMBlockBufferCopyDataBytes(blockBuffer, 0, length, chunk) == kCMBlockBufferNoErr) {
+					size_t floatCount = length / sizeof(float);
+					for (size_t i = 0; i < floatCount; i++) {
+						int bucket = (int)(((sampleIndex + (long long)i) * numSamples) / totalSamples);
+						if (bucket >= numSamples) {
+							bucket = numSamples - 1;
+						}
+						float s = chunk[i];
+						sumSq[bucket] += s * s;
+						counts[bucket]++;
+					}
+					sampleIndex += (long long)floatCount;
+				}
+				free(chunk);
+			}
+		}
+		CFRelease(sampleBuffer);
+	}
+
+	float* amps = (float*)calloc(numSamples, sizeof(float));
+	if (!amps) {
+		free(sumSq);
+		free(counts);
+		return result;
+	}
+
+	for (int i = 0; i < numSamples; i++) {
+		if (counts[i] > 0) {
+			amps[i] = sqrtf(sumSq[i] / (float)counts[i]);
+		}
+	}
+	free(sumSq);
+	free(counts);
+
+	result.data = amps;
+	result.length = numSamples;
+	return result;
+}
+
 #if TARGET_OS_IPHONE
 ImageConversionResult HEICToJPEG(const char* inFilename) {
 	ImageConversionResult result = {NULL, 0};
@@ -121,6 +239,22 @@ import (
 	"github.com/keybase/client/go/chat/utils"
 )
 
+func getAudioAmps(basename string) []float64 {
+	cbasename := C.CString(basename)
+	defer C.free(unsafe.Pointer(cbasename))
+	result := C.GetAudioAmplitudes(cbasename, C.int(audioAmpsCount))
+	if result.length == 0 || result.data == nil {
+		return nil
+	}
+	defer C.free(unsafe.Pointer(result.data))
+	amps := make([]float64, int(result.length))
+	cData := (*[1 << 20]C.float)(unsafe.Pointer(result.data))[:int(result.length):int(result.length)]
+	for i, v := range cData {
+		amps[i] = float64(v)
+	}
+	return amps
+}
+
 func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader,
 	basename string, nvh types.NativeVideoHelper,
 ) (res *PreviewRes, err error) {
@@ -137,6 +271,12 @@ func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader,
 	}
 	log.Debug(ctx, "previewVideo: length: %d duration: %ds", result.imageLength, duration)
 	if result.imageLength == 0 {
+		// Audio-only files (e.g. M4A) have no video track so no thumbnail, but AVFoundation
+		// can still read their duration. Extract amplitude data for the waveform visualization.
+		if duration > 1 && isAudioExtension(basename) {
+			amps := getAudioAmps(basename)
+			return previewAudio(duration, amps)
+		}
 		return res, errors.New("no data returned from native")
 	}
 	localDat := make([]byte, result.imageLength)

go/chat/attachments/preview_native.go

@@ -0,0 +1,55 @@
+//go:build darwin || android
+// +build darwin android
+
+package attachments
+
+import (
+	"math"
+	"path/filepath"
+	"strings"
+)
+
+const audioAmpsCount = 60
+
+func isAudioExtension(basename string) bool {
+	switch strings.ToLower(filepath.Ext(basename)) {
+	case ".m4a", ".mp3", ".aac", ".ogg", ".flac", ".wav", ".opus", ".aiff", ".caf":
+		return true
+	}
+	return false
+}
+
+func normalizeAudioAmps(amps []float64) []float64 {
+	if len(amps) == 0 {
+		return make([]float64, audioAmpsCount)
+	}
+	return amps
+}
+
+// previewAudio generates a waveform preview image and packages amplitude data
+// for an audio-only file. amps are linear RMS values in [0,1].
+func previewAudio(duration int, amps []float64) (*PreviewRes, error) {
+	amps = normalizeAudioAmps(amps)
+	// audioVisualizer expects dB values; convert from linear RMS.
+	dbAmps := make([]float64, len(amps))
+	for i, a := range amps {
+		if a <= 0 {
+			dbAmps[i] = -80
+		} else {
+			dbAmps[i] = 20 * math.Log10(a)
+		}
+	}
+	v := newAudioVisualizer(dbAmps)
+	dat, width := v.visualize()
+	return &PreviewRes{
+		Source:         dat,
+		ContentType:    "image/png",
+		BaseWidth:      width,
+		BaseHeight:     v.height,
+		BaseDurationMs: duration,
+		BaseIsAudio:    true,
+		AudioAmps:      amps,
+		PreviewWidth:   width,
+		PreviewHeight:  v.height,
+	}, nil
+}

go/chat/attachments/preview_native_test.go

@@ -0,0 +1,22 @@
+//go:build darwin || android
+// +build darwin android
+
+package attachments
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPreviewAudioEmptyAmps(t *testing.T) {
+	res, err := previewAudio(1234, nil)
+	require.NoError(t, err)
+	require.NotNil(t, res)
+	require.True(t, res.BaseIsAudio)
+	require.Equal(t, 1234, res.BaseDurationMs)
+	require.Len(t, res.AudioAmps, audioAmpsCount)
+	require.NotEmpty(t, res.Source)
+	require.Positive(t, res.PreviewWidth)
+	require.Positive(t, res.PreviewHeight)
+}

go/chat/types/interfaces.go

@@ -444,6 +444,7 @@ type AttachmentUploader interface {
 
 type NativeVideoHelper interface {
 	ThumbnailAndDuration(ctx context.Context, filename string) ([]byte, int, error)
+	AudioAmps(ctx context.Context, filename string) ([]float64, error)
 }
 
 // ShareConversation holds data for donating a conversation to the iOS share sheet.

go/chat/types/types.go

@@ -563,6 +563,10 @@ func (d DummyNativeVideoHelper) ThumbnailAndDuration(ctx context.Context, filena
 	return nil, 0, nil
 }
 
+func (d DummyNativeVideoHelper) AudioAmps(ctx context.Context, filename string) ([]float64, error) {
+	return nil, nil
+}
+
 type UnfurlerTaskStatus int
 
 const (