rudimentary support for sound (#4)

Makefile

@@ -6,7 +6,7 @@ RDIR=res
 CXX=g++
 CXXFLAGS=-I$(IDIR) -Wall -g
 
-LIBS=-lSDL2 -lSDL2_ttf -lSDL2_image -lavutil -lavformat -lavcodec -lswscale
+LIBS=-lSDL2 -lSDL2_ttf -lSDL2_image -lavutil -lavformat -lavcodec -lswscale -lswresample
 
 #-lavutil -lavformat -lavcodec
 #$(pkg-config --libs --cflags libavformat)

include/olive-audio.h

@@ -4,7 +4,12 @@
 #include <stdint.h>
 //#include <SDL2/SDL.h>
 
-extern uint8_t audioBuffer[192000];
+const unsigned int audioSampleRate = 44100;
+const unsigned int audioChannels = 2;
+const unsigned long bufferSize = 192000;
+extern uint8_t audioBuffer[bufferSize];
+extern bool debugAud;
 extern void audioCallback(void* userdata, uint8_t* stream, int len);
+extern unsigned long bufferIndex;
 
 #endif

include/olive-compositor.h

@@ -13,6 +13,7 @@ class Compositor {
 	public:
 		Compositor();
 		void compose(Sequence* sequence, SDL_Texture* canvas, long frame, bool display);
+		unsigned long recordedBufferIndex;
 	private:
 		void retrieveNextFrame(Clip* c, AVFrame* f);
 		//uint8_t* data;

include/sources-media.h

@@ -36,6 +36,7 @@ class Stream {
 		bool cached;
 		unsigned long lastFrame;
 		AVFrame* cacheFrame;
+		unsigned long cachedDataSize;
 };
 
 #endif

src/olive-audio.cpp

@@ -1,17 +1,40 @@
 #include "olive-audio.h"
+#include <iostream>
 #include <stdio.h>
 #include <SDL2/SDL.h>
 #include <math.h>
 
-uint8_t audioBuffer[192000];
+uint8_t audioBuffer[bufferSize];
+bool debugAud = true;
+unsigned long bufferIndex = 0;
 
 int sine = 0;
 
 void audioCallback(void* userdata, uint8_t* stream, int len) {
-	//stream = audioBuffer;
-	for (int i=0;i<len;i+=2) {
-		stream[i] = stream[i+1] = sin(sine * 0.025) * 64;
-		sine++;
-		//stream[i] = sin(i);
+	/*//stream = audioBuffer;
+	int copyLen = len;
+	if (bufferIndex + copyLen > bufferSize) {
+		copyLen = bufferSize - bufferIndex;
 	}
+	
+	//memset(audioBuffer+bufferIndex, 0, len);
+	bufferIndex += len;
+	if (bufferIndex >= bufferSize) {
+		bufferIndex = 0;
+		memcpy(stream+copyLen, audioBuffer+bufferIndex, len-copyLen);
+		//memset(audioBuffer+bufferIndex, 0, len-copyLen);
+	}*/
+
+	int copyLen = std::min((unsigned long) len, bufferSize - bufferIndex);
+
+	memcpy(stream, audioBuffer+bufferIndex, copyLen);
+	memset(audioBuffer+bufferIndex, 0, copyLen);
+
+	if (copyLen < len) {
+		int lastLen = len - copyLen;
+		memcpy(stream + copyLen, audioBuffer, lastLen);
+		memset(audioBuffer, 0, lastLen);
+	}
+
+	bufferIndex = (bufferIndex + len)%bufferSize;
 }

src/olive-compositor.cpp

@@ -12,16 +12,19 @@
 #include <tgmath.h>
 #endif
 
+//#define SWR_FLAG_RESAMPLE 0
+
 extern "C" {
 	#include <libavutil/avutil.h>
 	#include <libavutil/imgutils.h>
 	#include <libavformat/avformat.h>
 	#include <libavcodec/avcodec.h>
 	#include <libswscale/swscale.h>
+	#include <libswresample/swresample.h>
 }
 
 Compositor::Compositor() {
-	cachedSequenceWidth = cachedSequenceHeight = 0;
+	cachedSequenceWidth = cachedSequenceHeight = recordedBufferIndex = 0;
 	cached = false;
 }
 
@@ -86,7 +89,7 @@ void Compositor::compose(Sequence* sequence, SDL_Texture* canvas, long frame, bo
 		}
 
 		if (c->stream->avstream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
-			/*	
+			/*
 			// transform
 			transformPosX = 0;
 			transformPosY = 0;
@@ -213,14 +216,46 @@ void Compositor::compose(Sequence* sequence, SDL_Texture* canvas, long frame, bo
 				delete [] clipVPlane;
 			}
 			*/
+
 			printf("[WARNING] Video playback temporarily disabled to focus on audio.\n");
 		} else if (c->stream->avstream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
-			// TODO optimize
-			retrieveNextFrame(c, c->stream->cacheFrame);
-			int data_size = av_samples_get_buffer_size(NULL, c->stream->codecCtx->channels, c->stream->cacheFrame->nb_samples, c->stream->codecCtx->sample_fmt, 1);
-			int buf_size = 1024;
-			//assert(data_size <= buf_size);
-			memcpy(audioBuffer, c->stream->cacheFrame->data[0], buf_size);
+			// query whether the audio queue needs more samples
+			if (c->stream->cachedDataSize == 0 || recordedBufferIndex < (bufferIndex + c->stream->cachedDataSize)) {
+				struct SwrContext* swr_ctx = swr_alloc_set_opts(NULL, AV_CH_LAYOUT_STEREO, AV_SAMPLE_FMT_S16, audioSampleRate, c->stream->codecCtx->channel_layout, c->stream->codecCtx->sample_fmt, c->stream->codecCtx->sample_rate, 0, NULL);
+				swr_init(swr_ctx);
+
+				unsigned long readSize = 0;				
+				while (readSize < 16384) {
+					// TODO add seek code here
+
+					retrieveNextFrame(c, c->stream->cacheFrame);
+					//c->stream->cachedDataSize = av_samples_get_buffer_size(NULL, c->stream->codecCtx->channels, c->stream->cacheFrame->nb_samples, , 1);
+
+					uint8_t* output;
+					int out_samples = av_rescale_rnd(swr_get_delay(swr_ctx, c->stream->codecCtx->sample_rate) + c->stream->cacheFrame->nb_samples, audioSampleRate, c->stream->codecCtx->sample_rate, AV_ROUND_UP);
+					// printf("in samples: %i, out_samples: %i\n", c->stream->cacheFrame->nb_samples, out_samples);
+					av_samples_alloc(&output, NULL, audioChannels, out_samples, AV_SAMPLE_FMT_S16, 0);
+
+					out_samples = swr_convert(swr_ctx, &output, out_samples, (const uint8_t**) &c->stream->cacheFrame->data[0], c->stream->cacheFrame->nb_samples);
+
+					unsigned long byteLen = av_samples_get_buffer_size(NULL, audioChannels, out_samples, AV_SAMPLE_FMT_S16, 0);
+
+					unsigned long len = std::min(byteLen, bufferSize - recordedBufferIndex);
+					// printf("len: %lu\n", len);
+					memcpy(audioBuffer+recordedBufferIndex, output, len);
+					if (len < (unsigned long) byteLen) {
+						memcpy(audioBuffer, output+len, byteLen - len);
+					}
+
+					recordedBufferIndex = (recordedBufferIndex + byteLen)%bufferSize;
+					readSize += byteLen;
+					c->stream->cachedDataSize = byteLen;
+
+					av_freep(&output);
+				}
+
+				swr_free(&swr_ctx);			
+			}
 		} else {
 			printf("[ERROR] Clip %i contained an unrecognized type.\n", i);
 		}
@@ -252,7 +287,7 @@ void Compositor::retrieveNextFrame(Clip* c, AVFrame* f) {
 					printf("[ERROR] Failed to send packet to decoder.\n");
 				} else {
 					if (avcodec_receive_frame(c->stream->codecCtx, f) < 0) {
-						printf("[ERROR] Failed to receive packet from decoder.\n");
+						printf("[ERROR] Failed to receive packet from decoder\n");
 					} else {
 						frameFinished = true;
 					}

src/olive-timeline.cpp

@@ -103,14 +103,14 @@ Timeline::Timeline(Core* c) : Panel(c) {
 	sequence.frameRate = 59.94;
 	sequence.width = 1920;
 	sequence.height = 1080;
-	sequence.audioFreq = 48000;
+	sequence.audioFreq = audioSampleRate;
 	sequence.audioChannels = 2;
 
 	// TODO is this the best place for this?
 	// NOTE not causing segfault
 	SDL_AudioSpec targetSpec, actualSpec;
 
-	targetSpec.freq = sequence.audioFreq;
+	targetSpec.freq = audioSampleRate;
 	targetSpec.format = AUDIO_S16SYS;
 	targetSpec.channels = sequence.audioChannels;
 	targetSpec.silence = 0;
@@ -125,7 +125,6 @@ Timeline::Timeline(Core* c) : Panel(c) {
 		printf("%i. %s\n", i, SDL_GetAudioDeviceName(i, 0));
 	}
 	audioDevice = SDL_OpenAudioDevice(NULL, 0, &targetSpec, &actualSpec, 0);
-	printf("Chose: %i\n", audioDevice);
 	if (audioDevice == 0) {
 		printf("[ERROR] Could not open audio device. %s\n", SDL_GetError());
 	} else {
@@ -309,11 +308,17 @@ void Timeline::play() {
 	time = SDL_GetTicks();
 	playing = true;
 	viewer->setPauseText();
+
+	// audio buffer
+	compositor.recordedBufferIndex = bufferIndex;
 }
 
 void Timeline::pause() {
 	playing = false;
 	viewer->setPlayText();
+
+	// audio buffer
+	memset(audioBuffer, 0, bufferSize);
 }
 
 bool Timeline::isPlaying() {

src/sources-media.cpp

@@ -9,5 +9,5 @@ long Media::getLengthInFrames(double rate) {
 
 Stream::Stream() {	
 	cached = false;
-	lastFrame = 0;
+	lastFrame = cachedDataSize = 0;
 }