Permalink
Browse files

added support for merging all audio tracks from multiple movies

  • Loading branch information...
tuo committed Jan 10, 2015
1 parent eecb459 commit 4a3be64c6f1c0a44316045b71d70dd1714baa70c
@@ -56,6 +56,7 @@
- (void)startProcessing;
- (void)endProcessing;
- (void)cancelProcessing;
- (void)processMovieFrame:(CMSampleBufferRef)movieSampleBuffer;
- (void)processMovieFrame:(CMSampleBufferRef)movieSampleBuffer;
- (void)loadAsset:(dispatch_group_t)readyGroup;
@end
@@ -151,28 +151,36 @@ - (void)enableSynchronizedEncodingUsingMovieWriter:(GPUImageMovieWriter *)movieW
movieWriter.encodingLiveVideo = NO;
}
- (void)startProcessing
{
- (void)loadAsset:(dispatch_group_t)readyGroup {
NSAssert(self.url || self.asset, @"Url or asset should be passed for initial");
NSAssert(readyGroup, @"should pass a ready group to tell when i'm ready");
dispatch_group_enter(readyGroup);
/*
if( self.playerItem ) {
[self processPlayerItem];
return;
}
if(self.url == nil)
*/
if(self.url == nil && self.asset != nil)
{
[self processAsset];
return;
//directly inited from outside
dispatch_group_leave(readyGroup);
return;
}
if (_shouldRepeat) keepLooping = YES;
previousFrameTime = kCMTimeZero;
previousActualFrameTime = CFAbsoluteTimeGetCurrent();
NSDictionary *inputOptions = [NSDictionary dictionaryWithObject:[NSNumber numberWithBool:YES] forKey:AVURLAssetPreferPreciseDurationAndTimingKey];
AVURLAsset *inputAsset = [[AVURLAsset alloc] initWithURL:self.url options:inputOptions];
GPUImageMovie __block *blockSelf = self;
[inputAsset loadValuesAsynchronouslyForKeys:[NSArray arrayWithObject:@"tracks"] completionHandler: ^{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
NSError *error = nil;
@@ -181,11 +189,27 @@ - (void)startProcessing
{
return;
}
NSLog(@"GPUImageMovie %@ loaded successfully", blockSelf.url.lastPathComponent);
blockSelf.asset = inputAsset;
[blockSelf processAsset];
blockSelf = nil;
dispatch_group_leave(readyGroup);
});
}];
}
- (void)startProcessing
{
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
if( self.playerItem ) {
[self processPlayerItem];
return;
}
//NSAssert(self.asset, @"asset should be inited before use");
[self processAsset];
});
}
- (AVAssetReader*)createAssetReader
@@ -758,4 +782,5 @@ - (BOOL)videoEncodingIsFinished {
return videoEncodingIsFinished;
}
@end
@@ -22,7 +22,11 @@ extern NSString *const kGPUImageColorSwizzlingFragmentShaderString;
AVAssetWriterInput *assetWriterAudioInput;
AVAssetWriterInput *assetWriterVideoInput;
AVAssetWriterInputPixelBufferAdaptor *assetWriterPixelBufferInput;
AVAssetReader *assetAudioReader;
AVAssetReaderAudioMixOutput *assetAudioReaderTrackOutput;
GPUImageContext *_movieWriterContext;
CVPixelBufferRef renderTarget;
CVOpenGLESTextureRef renderTexture;
@@ -64,4 +68,11 @@ extern NSString *const kGPUImageColorSwizzlingFragmentShaderString;
- (void)processAudioBuffer:(CMSampleBufferRef)audioBuffer;
- (void)enableSynchronizationCallbacks;
- (void)setupAudioReaderWithTracks:(NSMutableArray *)audioTracks;
- (void)startAudioWritingWithComplectionBlock:(void (^)())pFunction;
- (void)startAudioRecording;
- (void)finishVideoRecordingWithCompletionHandler:(void (^)(void))handler;
@end
@@ -275,7 +275,7 @@ - (void)startRecording;
}
});
isRecording = YES;
// [assetWriter startSessionAtSourceTime:kCMTimeZero];
[assetWriter startSessionAtSourceTime:kCMTimeZero];
}
- (void)startRecordingInOrientation:(CGAffineTransform)orientationTransform;
@@ -310,6 +310,19 @@ - (void)cancelRecording;
});
}
- (void)finishVideoRecordingWithCompletionHandler:(void (^)(void))handler{
runSynchronouslyOnContextQueue(_movieWriterContext, ^{
isRecording = NO;
videoEncodingIsFinished = YES;
[assetWriterVideoInput markAsFinished];
if (handler)
runAsynchronouslyOnContextQueue(_movieWriterContext, handler);
});
}
- (void)finishRecording;
{
[self finishRecordingWithCompletionHandler:NULL];
@@ -440,7 +453,7 @@ - (void)processAudioBuffer:(CMSampleBufferRef)audioBuffer;
}
else
{
//NSLog(@"Wrote an audio frame %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, currentSampleTime)));
NSLog(@"Wrote an audio frame %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, currentSampleTime)));
}
if (_shouldInvalidateAudioSampleWhenDone)
@@ -744,12 +757,17 @@ - (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex;
}
else if(self.assetWriter.status == AVAssetWriterStatusWriting)
{
if (![assetWriterPixelBufferInput appendPixelBuffer:pixel_buffer withPresentationTime:frameTime])
if (![assetWriterPixelBufferInput appendPixelBuffer:pixel_buffer withPresentationTime:frameTime]){
NSLog(@"Problem appending pixel buffer at time: %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, frameTime)));
}
else
}else{
//NSLog(@"Wrote a video frame: %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, frameTime)));
}
} else
{
NSLog(@"Couldn't write a frame");
//NSLog(@"Couldn't write a frame");
//NSLog(@"Wrote a video frame: %@", CFBridgingRelease(CMTimeCopyDescription(kCFAllocatorDefault, frameTime)));
}
CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
@@ -931,4 +949,88 @@ - (AVAssetWriter*)assetWriter {
return assetWriter;
}
- (void)setupAudioReaderWithTracks:(NSMutableArray *)audioTracks {
if(audioTracks.count > 0){
AVMutableComposition* mixComposition = [AVMutableComposition composition];
for(AVAssetTrack *track in audioTracks){
NSLog(@"track url: %@ duration: %.2f", track.asset, CMTimeGetSeconds(track.asset.duration));
AVMutableCompositionTrack *compositionCommentaryTrack = [mixComposition addMutableTrackWithMediaType:AVMediaTypeAudio
preferredTrackID:kCMPersistentTrackID_Invalid];
[compositionCommentaryTrack insertTimeRange:CMTimeRangeMake(kCMTimeZero, track.asset.duration)
ofTrack:track
atTime:kCMTimeZero error:nil];
}
assetAudioReader = [AVAssetReader assetReaderWithAsset:mixComposition error:nil];
assetAudioReaderTrackOutput =
[[AVAssetReaderAudioMixOutput alloc] initWithAudioTracks:[mixComposition tracksWithMediaType:AVMediaTypeAudio]
audioSettings:nil];
[assetAudioReader addOutput:assetAudioReaderTrackOutput];
}
}
- (void)startAudioWritingWithComplectionBlock:(void (^)())completionBlock {
if(assetAudioReader){
NSLog(@"asset audio reader is prepared, kick off writing audio...");
audioQueue = dispatch_queue_create("com.sunsetlakesoftware.GPUImage.audioReadingQueue", NULL);
[assetWriterAudioInput requestMediaDataWhenReadyOnQueue:audioQueue usingBlock:^{
// Because the block is called asynchronously, check to see whether its task is complete.
BOOL completedOrFailed = NO;
// If the task isn't complete yet, make sure that the input is actually ready for more media data.
while ([assetWriterAudioInput isReadyForMoreMediaData] && !completedOrFailed) {
// Get the next audio sample buffer, and append it to the output file.
CMSampleBufferRef sampleBuffer = [assetAudioReaderTrackOutput copyNextSampleBuffer];
if (sampleBuffer != NULL) {
BOOL success = [assetWriterAudioInput appendSampleBuffer:sampleBuffer];
// if (success) {
// NSLog(@"append audio buffer success");
// } else {
// NSLog(@"append audio buffer failed");
// }
CFRelease(sampleBuffer);
sampleBuffer = NULL;
completedOrFailed = !success;
}
else {
completedOrFailed = YES;
}
}//end of loop
if (completedOrFailed) {
NSLog(@"audio wrint done");
// Mark the input as finished, but only if we haven't already done so, and then leave the dispatch group (since the audio work has finished).
audioEncodingIsFinished = YES;
[assetWriterAudioInput markAsFinished];
if(completionBlock){
completionBlock();
}
}
}];
} else{
NSLog(@" no audio reader is being set, this could happen when no audio tracks being set");
if(completionBlock){
completionBlock();
}
}
}
- (void)startAudioRecording {
if(assetAudioReader){
if(![assetAudioReader startReading]){
NSLog(@"asset audio reader start reading failed: %@", assetAudioReader.error);
}
}
}
@end

1 comment on commit 4a3be64

@tuo

This comment has been minimized.

Show comment
Hide comment
@tuo

tuo Jan 10, 2015

Owner

GPUImage Movie Writer - Added Multiple Audio Track Writing Support

See more detail on my blog: GPUImage Movie Writer: Merging All Audio Tracks From Multiple Movies

Here is a quick sample code to get it run:

self.gpuMovieA = [[GPUImageMovie alloc] initWithURL:rawVideoURL];    
self.gpuMovieFX = [[GPUImageMovie alloc] initWithURL:fxURL];

self.filter = [[GPUImageChromaKeyBlendFilter alloc] init];
[self.gpuMovieFX addTarget:self.filter];
[self.gpuMovieA addTarget:self.filter];

//setup writer
NSString *pathToMovie = [NSHomeDirectory() stringByAppendingPathComponent:@"Documents/gpu_output.mov"];
unlink([pathToMovie UTF8String]); // If a file already exists, AVAssetWriter won't let you record new frames, so delete the old movie
self.outputURL = [NSURL fileURLWithPath:pathToMovie];
self.movieWriter =  [[GPUImageMovieWriter alloc] initWithMovieURL:self.outputURL size:CGSizeMake(640.0/2, 640.0/2)];
[self.filter addTarget:self.movieWriter];

NSArray *movies = @[self.gpuMovieA, self.gpuMovieFX];
dispatch_group_t movieReadyDispatchGroup = dispatch_group_create();
for(GPUImageMovie *movie in movies){
    [movie loadAsset:movieReadyDispatchGroup];
}

__weak typeof(self) weakSelf = self;
dispatch_group_notify(movieReadyDispatchGroup, dispatch_get_main_queue(), ^{
    NSLog(@"all movies are ready to process :)");
    NSMutableArray *audioTracks = [NSMutableArray array];
    for(GPUImageMovie *movie in movies){
        AVAssetTrack *track = [movie.asset tracksWithMediaType:AVMediaTypeAudio].firstObject;
        if(track){
            [audioTracks addObject:track];
        }
    }

    if(audioTracks.count > 0){
        [self.movieWriter setupAudioReaderWithTracks:audioTracks];
        [self.movieWriter setHasAudioTrack:YES]; //use default audio settings, setup asset writer audio
    }

    self.recordSyncingDispatchGroup = dispatch_group_create();

    //this has to be called before, to make sure all audio/video in movie writer is set
    [self.movieWriter startRecording];
    [self.movieWriter startAudioRecording];

    //video handling
    dispatch_group_enter(self.recordSyncingDispatchGroup);
    [self.gpuMovieA startProcessing];
    [self.gpuMovieFX startProcessing];
    [self.movieWriter setCompletionBlock:^{
        [weakSelf.gpuMovieFX endProcessing];
        [weakSelf.gpuMovieA endProcessing];
        [weakSelf.movieWriter finishVideoRecordingWithCompletionHandler:^{
            NSLog(@"===video wrote is done");
            dispatch_group_leave(weakSelf.recordSyncingDispatchGroup);
        }];
    }];

    //audio handling
    dispatch_group_enter(self.recordSyncingDispatchGroup);
    [self.movieWriter startAudioRecording];
    [self.movieWriter startAudioWritingWithComplectionBlock:^{
        NSLog(@"====audio wring is done");
        dispatch_group_leave(weakSelf.recordSyncingDispatchGroup);
    }];

    dispatch_group_notify(self.recordSyncingDispatchGroup, dispatch_get_main_queue(), ^{
        NSLog(@"vidoe and audio writing are both done-----------------");
        [self.movieWriter finishRecordingWithCompletionHandler:^{
            NSLog(@"final clean up is done :)");
        }];
    });

});
Owner

tuo commented on 4a3be64 Jan 10, 2015

GPUImage Movie Writer - Added Multiple Audio Track Writing Support

See more detail on my blog: GPUImage Movie Writer: Merging All Audio Tracks From Multiple Movies

Here is a quick sample code to get it run:

self.gpuMovieA = [[GPUImageMovie alloc] initWithURL:rawVideoURL];    
self.gpuMovieFX = [[GPUImageMovie alloc] initWithURL:fxURL];

self.filter = [[GPUImageChromaKeyBlendFilter alloc] init];
[self.gpuMovieFX addTarget:self.filter];
[self.gpuMovieA addTarget:self.filter];

//setup writer
NSString *pathToMovie = [NSHomeDirectory() stringByAppendingPathComponent:@"Documents/gpu_output.mov"];
unlink([pathToMovie UTF8String]); // If a file already exists, AVAssetWriter won't let you record new frames, so delete the old movie
self.outputURL = [NSURL fileURLWithPath:pathToMovie];
self.movieWriter =  [[GPUImageMovieWriter alloc] initWithMovieURL:self.outputURL size:CGSizeMake(640.0/2, 640.0/2)];
[self.filter addTarget:self.movieWriter];

NSArray *movies = @[self.gpuMovieA, self.gpuMovieFX];
dispatch_group_t movieReadyDispatchGroup = dispatch_group_create();
for(GPUImageMovie *movie in movies){
    [movie loadAsset:movieReadyDispatchGroup];
}

__weak typeof(self) weakSelf = self;
dispatch_group_notify(movieReadyDispatchGroup, dispatch_get_main_queue(), ^{
    NSLog(@"all movies are ready to process :)");
    NSMutableArray *audioTracks = [NSMutableArray array];
    for(GPUImageMovie *movie in movies){
        AVAssetTrack *track = [movie.asset tracksWithMediaType:AVMediaTypeAudio].firstObject;
        if(track){
            [audioTracks addObject:track];
        }
    }

    if(audioTracks.count > 0){
        [self.movieWriter setupAudioReaderWithTracks:audioTracks];
        [self.movieWriter setHasAudioTrack:YES]; //use default audio settings, setup asset writer audio
    }

    self.recordSyncingDispatchGroup = dispatch_group_create();

    //this has to be called before, to make sure all audio/video in movie writer is set
    [self.movieWriter startRecording];
    [self.movieWriter startAudioRecording];

    //video handling
    dispatch_group_enter(self.recordSyncingDispatchGroup);
    [self.gpuMovieA startProcessing];
    [self.gpuMovieFX startProcessing];
    [self.movieWriter setCompletionBlock:^{
        [weakSelf.gpuMovieFX endProcessing];
        [weakSelf.gpuMovieA endProcessing];
        [weakSelf.movieWriter finishVideoRecordingWithCompletionHandler:^{
            NSLog(@"===video wrote is done");
            dispatch_group_leave(weakSelf.recordSyncingDispatchGroup);
        }];
    }];

    //audio handling
    dispatch_group_enter(self.recordSyncingDispatchGroup);
    [self.movieWriter startAudioRecording];
    [self.movieWriter startAudioWritingWithComplectionBlock:^{
        NSLog(@"====audio wring is done");
        dispatch_group_leave(weakSelf.recordSyncingDispatchGroup);
    }];

    dispatch_group_notify(self.recordSyncingDispatchGroup, dispatch_get_main_queue(), ^{
        NSLog(@"vidoe and audio writing are both done-----------------");
        [self.movieWriter finishRecordingWithCompletionHandler:^{
            NSLog(@"final clean up is done :)");
        }];
    });

});
Please sign in to comment.