28 #include "../include/FrameMapper.h" 34 reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), timeline_frame_offset(0)
76 throw ReaderClosed(
"No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.",
"");
79 void FrameMapper::AddField(int64_t frame)
82 AddField(
Field(frame, field_toggle));
85 void FrameMapper::AddField(
Field field)
91 field_toggle = (field_toggle ? false :
true);
98 void FrameMapper::Init()
100 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Init (Calculate frame mappings)",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
119 if ((fabs(original.
ToFloat() - 24.0) < 1
e-7 || fabs(original.
ToFloat() - 25.0) < 1
e-7 || fabs(original.
ToFloat() - 30.0) < 1
e-7) &&
120 (fabs(target.
ToFloat() - 24.0) < 1
e-7 || fabs(target.
ToFloat() - 25.0) < 1
e-7 || fabs(target.
ToFloat() - 30.0) < 1
e-7)) {
123 float difference = target.
ToInt() - original.
ToInt();
126 int field_interval = 0;
127 int frame_interval = 0;
131 field_interval = round(fabs(original.
ToInt() / difference));
134 frame_interval = field_interval * 2.0f;
143 for (int64_t field = 1; field <= number_of_fields; field++)
151 else if (difference > 0)
161 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
168 AddField(
Field(frame + 1, field_toggle));
170 else if (pulldown ==
PULLDOWN_NONE && field % frame_interval == 0)
177 else if (difference < 0)
183 field_toggle = (field_toggle ? false :
true);
185 else if (pulldown ==
PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
190 else if (pulldown ==
PULLDOWN_NONE && frame % field_interval == 0)
203 if (field % 2 == 0 && field > 0)
219 for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
222 AddField(rate_curve.
GetInt(frame_num));
223 AddField(rate_curve.
GetInt(frame_num));
232 int64_t start_samples_frame = 1;
233 int start_samples_position = 0;
235 for (int64_t field = 1; field <=
fields.size(); field++)
241 if (field % 2 == 0 && field > 0)
244 int64_t frame_number = field / 2 + timeline_frame_offset;
255 int64_t end_samples_frame = start_samples_frame;
256 int end_samples_position = start_samples_position;
259 while (remaining_samples > 0)
265 if (original_samples >= remaining_samples)
268 end_samples_position += remaining_samples - 1;
269 remaining_samples = 0;
273 end_samples_frame += 1;
274 end_samples_position = 0;
275 remaining_samples -= original_samples;
285 start_samples_frame = end_samples_frame;
286 start_samples_position = end_samples_position + 1;
289 start_samples_frame += 1;
290 start_samples_position = 0;
318 frame.
Odd.
Frame = TargetFrameNumber;
328 if(TargetFrameNumber < 1 ||
frames.size() == 0)
332 else if (TargetFrameNumber >
frames.size())
334 TargetFrameNumber =
frames.size();
337 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetMappedFrame",
"TargetFrameNumber", TargetFrameNumber,
"frames.size()",
frames.size(),
"frames[...].Odd",
frames[TargetFrameNumber - 1].Odd.Frame,
"frames[...].Even",
frames[TargetFrameNumber - 1].Even.Frame,
"", -1,
"", -1);
340 return frames[TargetFrameNumber - 1];
344 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
346 std::shared_ptr<Frame> new_frame;
353 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (from reader)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
359 new_frame = reader->
GetFrame(number);
373 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetOrCreateFrame (create blank)",
"number", number,
"samples_in_frame", samples_in_frame,
"", -1,
"", -1,
"", -1,
"", -1);
386 std::shared_ptr<Frame> final_frame = final_cache.
GetFrame(requested_frame);
387 if (final_frame)
return final_frame;
398 final_frame = final_cache.
GetFrame(requested_frame);
399 if (final_frame)
return final_frame;
403 int minimum_frames = 1;
406 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (Loop through frames)",
"requested_frame", requested_frame,
"minimum_frames", minimum_frames,
"", -1,
"", -1,
"", -1,
"", -1);
409 for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
413 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::GetFrame (inside omp for loop)",
"frame_number", frame_number,
"minimum_frames", minimum_frames,
"requested_frame", requested_frame,
"", -1,
"", -1,
"", -1);
417 std::shared_ptr<Frame> mapped_frame;
420 mapped_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
423 int channels_in_frame = mapped_frame->GetAudioChannelsCount();
424 int samples_in_frame =
Frame::GetSamplesPerFrame(frame_number + timeline_frame_offset, target, mapped_frame->SampleRate(), channels_in_frame);
433 info.
channels == mapped_frame->GetAudioChannelsCount() &&
435 mapped.
Samples.
total == mapped_frame->GetAudioSamplesCount() &&
438 mapped_frame->number == frame_number &&
442 final_cache.
Add(mapped_frame);
447 std::shared_ptr<Frame> frame = std::make_shared<Frame>(frame_number, 1, 1,
"#000000", samples_in_frame, channels_in_frame);
448 frame->SampleRate(mapped_frame->SampleRate());
449 frame->ChannelsLayout(mapped_frame->ChannelsLayout());
453 std::shared_ptr<Frame> odd_frame;
454 odd_frame = GetOrCreateFrame(mapped.
Odd.
Frame);
457 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*odd_frame->GetImage())),
true);
460 std::shared_ptr<Frame> even_frame;
461 even_frame = GetOrCreateFrame(mapped.
Even.
Frame);
463 frame->AddImage(std::shared_ptr<QImage>(
new QImage(*even_frame->GetImage())),
false);
467 bool need_resampling =
false;
473 need_resampling =
true;
484 const int EXTRA_INPUT_SAMPLES = 20;
487 copy_samples.
sample_end += EXTRA_INPUT_SAMPLES;
488 int samples_per_end_frame =
491 if (copy_samples.
sample_end >= samples_per_end_frame)
495 copy_samples.
sample_end -= samples_per_end_frame;
497 copy_samples.
total += EXTRA_INPUT_SAMPLES;
504 int samples_per_start_frame =
507 if (copy_samples.
sample_start >= samples_per_start_frame)
513 copy_samples.
total -= EXTRA_INPUT_SAMPLES;
518 int samples_copied = 0;
523 int remaining_samples = copy_samples.
total - samples_copied;
524 int number_to_copy = 0;
527 std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
528 int original_samples = original_frame->GetAudioSamplesCount();
531 for (
int channel = 0; channel < channels_in_frame; channel++)
536 number_to_copy = original_samples - copy_samples.
sample_start;
537 if (number_to_copy > remaining_samples)
538 number_to_copy = remaining_samples;
541 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.
sample_start, number_to_copy, 1.0);
546 number_to_copy = original_samples;
547 if (number_to_copy > remaining_samples)
548 number_to_copy = remaining_samples;
551 frame->AddAudio(
true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
557 if (number_to_copy > remaining_samples)
558 number_to_copy = remaining_samples;
561 frame->AddAudio(
false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
566 samples_copied += number_to_copy;
576 final_cache.
Add(frame);
581 return final_cache.
GetFrame(requested_frame);
592 float difference = target.
ToInt() - original.
ToInt();
594 int field_interval = 0;
595 int frame_interval = 0;
600 field_interval = round(fabs(original.
ToInt() / difference));
603 frame_interval = field_interval * 2.0f;
607 for (
float map = 1; map <=
frames.size(); map++)
610 cout <<
"Target frame #: " << map <<
" mapped to original frame #:\t(" << frame.
Odd.
Frame <<
" odd, " << frame.
Even.
Frame <<
" even)" << endl;
631 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Open",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
646 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
653 avresample_close(avr);
654 avresample_free(&avr);
673 root[
"type"] =
"FrameMapper";
685 bool success = reader.parse( value, root );
688 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
698 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
719 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ChangeMapping",
"target_fps.num", target_fps.
num,
"target_fps.den", target_fps.
den,
"target_pulldown", target_pulldown,
"target_sample_rate", target_sample_rate,
"target_channels", target_channels,
"target_channel_layout", target_channel_layout);
725 target.
num = target_fps.
num;
726 target.
den = target_fps.
den;
731 pulldown = target_pulldown;
744 avresample_close(avr);
745 avresample_free(&avr);
756 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::SetTimelineFrameOffset",
"offset", offset,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
762 timeline_frame_offset = offset;
769 int total_frame_samples = 0;
770 int channels_in_frame = frame->GetAudioChannelsCount();
771 int sample_rate_in_frame = frame->SampleRate();
772 int samples_in_frame = frame->GetAudioSamplesCount();
773 ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
775 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio",
"frame->number", frame->number,
"original_frame_number", original_frame_number,
"channels_in_frame", channels_in_frame,
"samples_in_frame", samples_in_frame,
"sample_rate_in_frame", sample_rate_in_frame,
"", -1);
778 float* frame_samples_float = NULL;
780 frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
783 total_frame_samples = samples_in_frame * channels_in_frame;
786 int16_t* frame_samples = (int16_t*) av_malloc(
sizeof(int16_t)*total_frame_samples);
789 for (
int s = 0; s < total_frame_samples; s++)
791 frame_samples[s] =
int(frame_samples_float[s] * (1 << 15));
795 delete[] frame_samples_float;
796 frame_samples_float = NULL;
798 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (got sample data from frame)",
"frame->number", frame->number,
"total_frame_samples", total_frame_samples,
"target channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"target sample_rate",
info.
sample_rate,
"samples_in_frame", samples_in_frame);
804 audio_frame->nb_samples = total_frame_samples / channels_in_frame;
806 int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
807 audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
811 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio ERROR [" + (
string)
av_err2str(error_code) +
"]",
"error_code", error_code,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
812 throw ErrorEncodingVideo(
"Error while resampling audio in frame mapper", frame->number);
818 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (adjust # of samples)",
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"sample_rate_in_frame", sample_rate_in_frame,
"info.channels",
info.
channels,
"channels_in_frame", channels_in_frame,
"original_frame_number", original_frame_number);
823 audio_converted->nb_samples = total_frame_samples;
824 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
826 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (preparing for resample)",
"in_sample_fmt", AV_SAMPLE_FMT_S16,
"out_sample_fmt", AV_SAMPLE_FMT_S16,
"in_sample_rate", sample_rate_in_frame,
"out_sample_rate",
info.
sample_rate,
"in_channels", channels_in_frame,
"out_channels",
info.
channels);
832 avr = avresample_alloc_context();
833 av_opt_set_int(avr,
"in_channel_layout", channel_layout_in_frame, 0);
835 av_opt_set_int(avr,
"in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
836 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
837 av_opt_set_int(avr,
"in_sample_rate", sample_rate_in_frame, 0);
839 av_opt_set_int(avr,
"in_channels", channels_in_frame, 0);
841 avresample_open(avr);
845 nb_samples = avresample_convert(avr,
846 audio_converted->data,
847 audio_converted->linesize[0],
848 audio_converted->nb_samples,
850 audio_frame->linesize[0],
851 audio_frame->nb_samples);
854 int16_t* resampled_samples =
new int16_t[(nb_samples *
info.
channels)];
857 memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels));
860 av_freep(&audio_frame->data[0]);
862 av_freep(&audio_converted->data[0]);
864 frame_samples = NULL;
867 int channel_buffer_size = nb_samples;
870 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Audio successfully resampled)",
"nb_samples", nb_samples,
"total_frame_samples", total_frame_samples,
"info.sample_rate",
info.
sample_rate,
"channels_in_frame", channels_in_frame,
"info.channels",
info.
channels,
"info.channel_layout",
info.
channel_layout);
873 float *channel_buffer =
new float[channel_buffer_size];
876 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
879 for (
int z = 0; z < channel_buffer_size; z++)
880 channel_buffer[z] = 0.0f;
886 for (
int sample = 0; sample < (nb_samples *
info.
channels); sample++)
889 if (channel_filter == channel)
892 channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
908 frame->AddAudio(
true, channel_filter, 0, channel_buffer, position, 1.0f);
910 ZmqLogger::Instance()->
AppendDebugMethod(
"FrameMapper::ResampleMappedAudio (Add audio to channel)",
"number of samples", position,
"channel_filter", channel_filter,
"", -1,
"", -1,
"", -1,
"", -1);
918 delete[] channel_buffer;
919 channel_buffer = NULL;
922 delete[] resampled_samples;
923 resampled_samples = NULL;
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
int max_height
The maximium image height needed by this clip (used for optimizations)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
ReaderBase * Reader()
Get the current reader.
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
vector< MappedFrame > frames
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
~FrameMapper()
Destructor.
float duration
Length of time (in seconds)
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
This struct holds a single field (half a frame).
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
Exception when encoding audio packet.
void AddPoint(Point p)
Add a new point on the key-frame. Each point has a primary coordinate, a left handle, and a right handle.
This struct holds a the range of samples needed by this frame.
virtual std::shared_ptr< Frame > GetFrame(int64_t number)=0
void SetMaxSize(int width, int height)
Set Max Image Size (used for performance optimization)
bool has_audio
Determines if this file has an audio stream.
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
int height
The height of the video (in pixels)
#define AV_ALLOCATE_FRAME()
bool IsOpen()
Determine if reader is open or closed.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
bool has_single_image
Determines if this file only contains a single image.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
int GetInt(int64_t index)
Get the rounded INT value at a specific index.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
#define av_err2str(errnum)
ReaderInfo info
Information about the current media file.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
This struct holds two fields which together make up a complete video frame.
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
string Json()
Get and Set JSON methods.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
This namespace is the default namespace for all code in the openshot library.
Do not apply pull-down techniques, just repeat or skip entire frames.
Linear curves are angular, straight lines between two points.
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
PulldownType
This enumeration determines how frame rates are increased or decreased.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
A Keyframe is a collection of Point instances, which is used to vary a number or property over time...
int max_width
The maximum image width needed by this clip (used for optimizations)
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
void SetJson(string value)
Load JSON string into this object.
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Exception when too many seek attempts happen.
virtual bool IsOpen()=0
Determine if reader is open or closed.
void SetTimelineFrameOffset(int64_t offset)