From 91945f03dc3d1ee9848e542db5189bad97c640de Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 13 Oct 2020 17:08:27 -0500 Subject: [PATCH] Replacing audio fix implementation with ParentClip(), to access clip start and position (if any) --- include/FrameMapper.h | 12 +++--- src/FrameMapper.cpp | 85 +++++++++++++++++++++---------------- src/Timeline.cpp | 4 +- tests/FrameMapper_Tests.cpp | 85 ++++++------------------------------- 4 files changed, 67 insertions(+), 119 deletions(-) diff --git a/include/FrameMapper.h b/include/FrameMapper.h index 85c933d25..7f6c9c165 100644 --- a/include/FrameMapper.h +++ b/include/FrameMapper.h @@ -147,9 +147,6 @@ namespace openshot bool is_dirty; // When this is true, the next call to GetFrame will re-init the mapping SWRCONTEXT *avr; // Audio resampling context object - float position; - float start; - // Internal methods used by init void AddField(int64_t frame); void AddField(Field field); @@ -157,6 +154,9 @@ namespace openshot // Get Frame or Generate Blank Frame std::shared_ptr GetOrCreateFrame(int64_t number); + /// Adjust frame number for Clip position and start (which can result in a different number) + int64_t AdjustFrameNumber(int64_t clip_frame_number, float position, float start); + // Use the original and target frame rates and a pull-down technique to create // a mapping between the original fields and frames or a video to a new frame rate. // This might repeat or skip fields and frames of the original video, depending on @@ -169,13 +169,13 @@ namespace openshot std::vector frames; // List of all frames /// Default constructor for openshot::FrameMapper class - FrameMapper(ReaderBase *reader, Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout, float clipPosition, float clipStart); + FrameMapper(ReaderBase *reader, Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout); /// Destructor virtual ~FrameMapper(); /// Change frame rate or audio mapping details - void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout, float clipPosition, float clipStart); + void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout); /// Close the openshot::FrameMapper and internal reader void Close() override; @@ -220,8 +220,6 @@ namespace openshot /// Resample audio and map channels (if needed) void ResampleMappedAudio(std::shared_ptr frame, int64_t original_frame_number); - - int64_t ConvPositon(int64_t clip_frame_number); }; } diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index d05cf7f66..332211737 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -29,11 +29,12 @@ */ #include "../include/FrameMapper.h" +#include "../include/Clip.h" using namespace std; using namespace openshot; -FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout, float clipPosition, float clipStart) : +FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) : reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL) { // Set the original frame rate from the reader @@ -52,8 +53,6 @@ FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType targe info.width = reader->info.width; info.height = reader->info.height; - position = clipPosition; - start = clipStart; // Used to toggle odd / even fields field_toggle = true; @@ -118,6 +117,15 @@ void FrameMapper::Init() // Clear cache final_cache.Clear(); + // Get clip position from parent clip (if any) + float clipPosition = 0.0; + float clipStart = 0.0; + Clip *parent = (Clip *) ParentClip(); + if (parent) { + clipPosition = parent->Position(); + clipStart = parent->Start(); + } + // Some framerates are handled special, and some use a generic Keyframe curve to // map the framerates. These are the special framerates: if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) && @@ -259,12 +267,12 @@ void FrameMapper::Init() // the original sample rate. int64_t end_samples_frame = start_samples_frame; int end_samples_position = start_samples_position; - int remaining_samples = Frame::GetSamplesPerFrame(ConvPositon(frame_number), target, reader->info.sample_rate, reader->info.channels); + int remaining_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number, clipPosition, clipStart), target, reader->info.sample_rate, reader->info.channels); while (remaining_samples > 0) { // get original samples - int original_samples = Frame::GetSamplesPerFrame(ConvPositon(end_samples_frame), original, reader->info.sample_rate, reader->info.channels) - end_samples_position; + int original_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(end_samples_frame, clipPosition, clipStart), original, reader->info.sample_rate, reader->info.channels) - end_samples_position; // Enough samples if (original_samples >= remaining_samples) @@ -284,12 +292,12 @@ void FrameMapper::Init() // Create the sample mapping struct - SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(ConvPositon(frame_number), target, reader->info.sample_rate, reader->info.channels)}; + SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number, clipPosition, clipStart), target, reader->info.sample_rate, reader->info.channels)}; // Reset the audio variables start_samples_frame = end_samples_frame; start_samples_position = end_samples_position + 1; - if (start_samples_position >= Frame::GetSamplesPerFrame(ConvPositon(start_samples_frame), original, reader->info.sample_rate, reader->info.channels)) + if (start_samples_position >= Frame::GetSamplesPerFrame(AdjustFrameNumber(start_samples_frame, clipPosition, clipStart), original, reader->info.sample_rate, reader->info.channels)) { start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one) start_samples_position = 0; // reset to 0, since we wrapped @@ -355,8 +363,17 @@ std::shared_ptr FrameMapper::GetOrCreateFrame(int64_t number) { std::shared_ptr new_frame; + // Get clip position from parent clip (if any) + float clipPosition = 0.0; + float clipStart = 0.0; + Clip *parent = (Clip *) ParentClip(); + if (parent) { + clipPosition = parent->Position(); + clipStart = parent->Start(); + } + // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now) - int samples_in_frame = Frame::GetSamplesPerFrame(ConvPositon(number), target, reader->info.sample_rate, reader->info.channels); + int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(number, clipPosition, clipStart), target, reader->info.sample_rate, reader->info.channels); try { // Debug output @@ -406,6 +423,15 @@ std::shared_ptr FrameMapper::GetFrame(int64_t requested_frame) final_frame = final_cache.GetFrame(requested_frame); if (final_frame) return final_frame; + // Get clip position from parent clip (if any) + float clipPosition = 0.0; + float clipStart = 0.0; + Clip *parent = (Clip *) ParentClip(); + if (parent) { + clipPosition = parent->Position(); + clipStart = parent->Start(); + } + // Minimum number of frames to process (for performance reasons) // Dialing this down to 1 for now, as it seems to improve performance, and reduce export crashes int minimum_frames = 1; @@ -429,7 +455,7 @@ std::shared_ptr FrameMapper::GetFrame(int64_t requested_frame) // Get # of channels in the actual frame int channels_in_frame = mapped_frame->GetAudioChannelsCount(); - int samples_in_frame = Frame::GetSamplesPerFrame(ConvPositon(frame_number), target, mapped_frame->SampleRate(), channels_in_frame); + int samples_in_frame = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame_number, clipPosition, clipStart), target, mapped_frame->SampleRate(), channels_in_frame); // Determine if mapped frame is identical to source frame // including audio sample distribution according to mapped.Samples, @@ -596,21 +622,6 @@ void FrameMapper::PrintMapping() // Recalculate mappings Init(); - // Get the difference (in frames) between the original and target frame rates - float difference = target.ToInt() - original.ToInt(); - - int field_interval = 0; - int frame_interval = 0; - - if (difference != 0) - { - // Find the number (i.e. interval) of fields that need to be skipped or repeated - field_interval = round(fabs(original.ToInt() / difference)); - - // Get frame interval (2 fields per frame) - frame_interval = field_interval * 2.0f; - } - // Loop through frame mappings for (float map = 1; map <= frames.size(); map++) { @@ -621,7 +632,6 @@ void FrameMapper::PrintMapping() } - // Determine if reader is open or closed bool FrameMapper::IsOpen() { if (reader) @@ -630,7 +640,6 @@ bool FrameMapper::IsOpen() { return false; } - // Open the internal reader void FrameMapper::Open() { @@ -689,7 +698,6 @@ Json::Value FrameMapper::JsonValue() const { // Create root json object Json::Value root = ReaderBase::JsonValue(); // get parent properties root["type"] = "FrameMapper"; - root["position"] = position; // return JsonValue return root; @@ -718,10 +726,6 @@ void FrameMapper::SetJsonValue(const Json::Value root) { // Set parent data ReaderBase::SetJsonValue(root); - if(!root["position"].isNull()){ - position = root["position"].asDouble(); - } - // Re-Open path, and re-init everything (if needed) if (reader) { @@ -731,7 +735,7 @@ void FrameMapper::SetJsonValue(const Json::Value root) { } // Change frame rate or audio mapping details -void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout, float clipPosition, float clipStart) +void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) { ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ChangeMapping", "target_fps.num", target_fps.num, "target_fps.den", target_fps.den, "target_pulldown", target_pulldown, "target_sample_rate", target_sample_rate, "target_channels", target_channels, "target_channel_layout", target_channel_layout); @@ -750,9 +754,6 @@ void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldow info.channels = target_channels; info.channel_layout = target_channel_layout; - position = clipPosition; - start = clipStart; - // Clear cache final_cache.Clear(); @@ -775,6 +776,15 @@ void FrameMapper::ResampleMappedAudio(std::shared_ptr frame, int64_t orig // Recalculate mappings Init(); + // Get clip position from parent clip (if any) + float clipPosition = 0.0; + float clipStart = 0.0; + Clip *parent = (Clip *) ParentClip(); + if (parent) { + clipPosition = parent->Position(); + clipStart = parent->Start(); + } + // Init audio buffers / variables int total_frame_samples = 0; int channels_in_frame = frame->GetAudioChannelsCount(); @@ -836,7 +846,7 @@ void FrameMapper::ResampleMappedAudio(std::shared_ptr frame, int64_t orig } // Update total samples & input frame size (due to bigger or smaller data types) - total_frame_samples = Frame::GetSamplesPerFrame(ConvPositon(frame->number), target, info.sample_rate, info.channels); + total_frame_samples = Frame::GetSamplesPerFrame(AdjustFrameNumber(frame->number, clipPosition, clipStart), target, info.sample_rate, info.channels); ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (adjust # of samples)", "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "sample_rate_in_frame", sample_rate_in_frame, "info.channels", info.channels, "channels_in_frame", channels_in_frame, "original_frame_number", original_frame_number); @@ -946,7 +956,8 @@ void FrameMapper::ResampleMappedAudio(std::shared_ptr frame, int64_t orig resampled_samples = NULL; } -int64_t FrameMapper::ConvPositon(int64_t clip_frame_number){ +// Adjust frame number for Clip position and start (which can result in a different number) +int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number, float position, float start) { int64_t clip_start_frame = (start * info.fps.ToDouble()) + 1; int64_t clip_start_position = round(position * info.fps.ToDouble()) + 1; diff --git a/src/Timeline.cpp b/src/Timeline.cpp index e45a0991a..736bee156 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -348,14 +348,14 @@ void Timeline::apply_mapper_to_clip(Clip* clip) } else { // Create a new FrameMapper to wrap the current reader - FrameMapper* mapper = new FrameMapper(clip->Reader(), info.fps, PULLDOWN_NONE, info.sample_rate, info.channels, info.channel_layout, clip->Position(), clip->Start()); + FrameMapper* mapper = new FrameMapper(clip->Reader(), info.fps, PULLDOWN_NONE, info.sample_rate, info.channels, info.channel_layout); allocated_frame_mappers.insert(mapper); clip_reader = (ReaderBase*) mapper; } // Update the mapping FrameMapper* clip_mapped_reader = (FrameMapper*) clip_reader; - clip_mapped_reader->ChangeMapping(info.fps, PULLDOWN_NONE, info.sample_rate, info.channels, info.channel_layout, clip->Position(), clip->Start()); + clip_mapped_reader->ChangeMapping(info.fps, PULLDOWN_NONE, info.sample_rate, info.channels, info.channel_layout); // Update clip reader clip->Reader(clip_reader); diff --git a/tests/FrameMapper_Tests.cpp b/tests/FrameMapper_Tests.cpp index 02634c373..54e76e64c 100644 --- a/tests/FrameMapper_Tests.cpp +++ b/tests/FrameMapper_Tests.cpp @@ -42,7 +42,7 @@ TEST(FrameMapper_Get_Valid_Frame) DummyReader r(Fraction(24,1), 720, 480, 22000, 2, 5.0); // Create mapping between 24 fps and 29.97 fps using classic pulldown - FrameMapper mapping(&r, Fraction(30000, 1001), PULLDOWN_CLASSIC, 22000, 2, LAYOUT_STEREO, 0.0, 0.0); + FrameMapper mapping(&r, Fraction(30000, 1001), PULLDOWN_CLASSIC, 22000, 2, LAYOUT_STEREO); try { @@ -63,7 +63,7 @@ TEST(FrameMapper_Invalid_Frame_Too_Small) DummyReader r(Fraction(24,1), 720, 480, 22000, 2, 5.0); // Create mapping 24 fps and 29.97 fps - FrameMapper mapping(&r, Fraction(30000, 1001), PULLDOWN_CLASSIC, 22000, 2, LAYOUT_STEREO, 0.0, 0.0); + FrameMapper mapping(&r, Fraction(30000, 1001), PULLDOWN_CLASSIC, 22000, 2, LAYOUT_STEREO); // Check invalid frame number CHECK_THROW(mapping.GetMappedFrame(0), OutOfBoundsFrame); @@ -76,7 +76,7 @@ TEST(FrameMapper_24_fps_to_30_fps_Pulldown_Classic) DummyReader r(Fraction(24,1), 720, 480, 22000, 2, 5.0); // Create mapping 24 fps and 30 fps - FrameMapper mapping(&r, Fraction(30, 1), PULLDOWN_CLASSIC, 22000, 2, LAYOUT_STEREO, 0.0, 0.0); + FrameMapper mapping(&r, Fraction(30, 1), PULLDOWN_CLASSIC, 22000, 2, LAYOUT_STEREO); MappedFrame frame2 = mapping.GetMappedFrame(2); MappedFrame frame3 = mapping.GetMappedFrame(3); @@ -93,7 +93,7 @@ TEST(FrameMapper_24_fps_to_30_fps_Pulldown_Advanced) DummyReader r(Fraction(24,1), 720, 480, 22000, 2, 5.0); // Create mapping 24 fps and 30 fps - FrameMapper mapping(&r, Fraction(30, 1), PULLDOWN_ADVANCED, 22000, 2, LAYOUT_STEREO, 0.0, 0.0); + FrameMapper mapping(&r, Fraction(30, 1), PULLDOWN_ADVANCED, 22000, 2, LAYOUT_STEREO); MappedFrame frame2 = mapping.GetMappedFrame(2); MappedFrame frame3 = mapping.GetMappedFrame(3); MappedFrame frame4 = mapping.GetMappedFrame(4); @@ -113,7 +113,7 @@ TEST(FrameMapper_24_fps_to_30_fps_Pulldown_None) DummyReader r(Fraction(24,1), 720, 480, 22000, 2, 5.0); // Create mapping 24 fps and 30 fps - FrameMapper mapping(&r, Fraction(30, 1), PULLDOWN_NONE, 22000, 2, LAYOUT_STEREO, 0.0, 0.0); + FrameMapper mapping(&r, Fraction(30, 1), PULLDOWN_NONE, 22000, 2, LAYOUT_STEREO); MappedFrame frame4 = mapping.GetMappedFrame(4); MappedFrame frame5 = mapping.GetMappedFrame(5); @@ -130,7 +130,7 @@ TEST(FrameMapper_30_fps_to_24_fps_Pulldown_Classic) DummyReader r(Fraction(30, 1), 720, 480, 22000, 2, 5.0); // Create mapping between 30 fps and 24 fps - FrameMapper mapping(&r, Fraction(24, 1), PULLDOWN_CLASSIC, 22000, 2, LAYOUT_STEREO, 0.0, 0.0); + FrameMapper mapping(&r, Fraction(24, 1), PULLDOWN_CLASSIC, 22000, 2, LAYOUT_STEREO); MappedFrame frame3 = mapping.GetMappedFrame(3); MappedFrame frame4 = mapping.GetMappedFrame(4); MappedFrame frame5 = mapping.GetMappedFrame(5); @@ -150,7 +150,7 @@ TEST(FrameMapper_30_fps_to_24_fps_Pulldown_Advanced) DummyReader r(Fraction(30, 1), 720, 480, 22000, 2, 5.0); // Create mapping between 30 fps and 24 fps - FrameMapper mapping(&r, Fraction(24, 1), PULLDOWN_ADVANCED, 22000, 2, LAYOUT_STEREO, 0.0, 0.0); + FrameMapper mapping(&r, Fraction(24, 1), PULLDOWN_ADVANCED, 22000, 2, LAYOUT_STEREO); MappedFrame frame2 = mapping.GetMappedFrame(2); MappedFrame frame3 = mapping.GetMappedFrame(3); MappedFrame frame4 = mapping.GetMappedFrame(4); @@ -170,7 +170,7 @@ TEST(FrameMapper_30_fps_to_24_fps_Pulldown_None) DummyReader r(Fraction(30, 1), 720, 480, 22000, 2, 5.0); // Create mapping between 30 fps and 24 fps - FrameMapper mapping(&r, Fraction(24, 1), PULLDOWN_NONE, 22000, 2, LAYOUT_STEREO, 0.0, 0.0); + FrameMapper mapping(&r, Fraction(24, 1), PULLDOWN_NONE, 22000, 2, LAYOUT_STEREO); MappedFrame frame4 = mapping.GetMappedFrame(4); MappedFrame frame5 = mapping.GetMappedFrame(5); @@ -189,7 +189,7 @@ TEST(FrameMapper_resample_audio_48000_to_41000) FFmpegReader r(path.str()); // Map to 30 fps, 3 channels surround, 44100 sample rate - FrameMapper map(&r, Fraction(30,1), PULLDOWN_NONE, 44100, 3, LAYOUT_SURROUND, 0.0, 0.0); + FrameMapper map(&r, Fraction(30,1), PULLDOWN_NONE, 44100, 3, LAYOUT_SURROUND); map.Open(); // Check details @@ -199,7 +199,7 @@ TEST(FrameMapper_resample_audio_48000_to_41000) CHECK_EQUAL(1470, map.GetFrame(50)->GetAudioSamplesCount()); // Change mapping data - map.ChangeMapping(Fraction(25,1), PULLDOWN_NONE, 22050, 1, LAYOUT_MONO, 0.0, 0.0); + map.ChangeMapping(Fraction(25,1), PULLDOWN_NONE, 22050, 1, LAYOUT_MONO); // Check details CHECK_EQUAL(1, map.GetFrame(1)->GetAudioChannelsCount()); @@ -213,13 +213,12 @@ TEST(FrameMapper_resample_audio_48000_to_41000) TEST(FrameMapper_AudioSample_Distribution) { - CacheMemory cache; int OFFSET = 0; float AMPLITUDE = 0.2; double ANGLE = 0.0; int NUM_SAMPLES = 100; - std::cout << "Starting Resample Test" << std::endl; + //std::cout << "Starting Resample Test" << std::endl; for (int64_t frame_number = 1; frame_number <= 90; frame_number++) { @@ -283,7 +282,6 @@ TEST(FrameMapper_AudioSample_Distribution) // which overlapping Frame instances have different # of samples for the Timeline. // TODO: Moving to 0.0 position, to simplify this test for now - c2.Position(0.041666667 * 14); c2.Start(1.0); c2.End(10.0); @@ -295,7 +293,7 @@ TEST(FrameMapper_AudioSample_Distribution) std::string json_val = t1.Json(); - std::cout << json_val << std::endl; + //std::cout << json_val << std::endl; //t1.SetJson(t1.Json()); t1.Open(); @@ -309,7 +307,6 @@ TEST(FrameMapper_AudioSample_Distribution) // Open writer w.Open(); - w.WriteFrame(&t1, 5, 50); //for (int64_t frame_number = 1; frame_number <= 90; frame_number++){ @@ -328,62 +325,4 @@ TEST(FrameMapper_AudioSample_Distribution) cache.Clear(); r.Close(); - } - - -/* -TEST(FrameMapperVideoEdition){ - - stringstream path; - path << TEST_MEDIA_PATH << "baseline.mkv"; - FFmpegReader r(path.str()); - r.Open(); - - Clip c1; - c1.Reader(&r); - c1.Layer(1); - c1.Position(0.0); - c1.Start(0.0); - c1.End(45.0); - - Clip c2; - c2.Reader(&r); - c2.Layer(1); - c2.Position(30.0); - c2.Start(0.0); - c2.End(45.0); - - Timeline t1(1280, 720, Fraction(24, 1), 44100, 2, LAYOUT_STEREO); - t1.AddClip(&c1); - t1.AddClip(&c2); - - t1.Open(); - - - FFmpegWriter w("simple-edit.mp4"); - - // Set options - w.SetAudioOptions("aac", 44100, 192000); - w.SetVideoOptions("libx264", 1280, 720, Fraction(24,1), 5000000); - - // Open writer - w.Open(); - - - w.WriteFrame(&t1, 1, t1.GetMaxFrame()); - - // Close writer & reader - w.Close(); - - //map.Close(); - //map2.Close(); - - t1.Close(); - - - r.Close(); - - - -}*/ \ No newline at end of file