OpenShot Library | libopenshot  0.1.9
FrameMapper.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for the FrameMapper class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2014 OpenShot Studios, LLC
9  * <http://www.openshotstudios.com/>. This file is part of
10  * OpenShot Library (libopenshot), an open-source project dedicated to
11  * delivering high quality video editing and animation solutions to the
12  * world. For more information visit <http://www.openshot.org/>.
13  *
14  * OpenShot Library (libopenshot) is free software: you can redistribute it
15  * and/or modify it under the terms of the GNU Lesser General Public License
16  * as published by the Free Software Foundation, either version 3 of the
17  * License, or (at your option) any later version.
18  *
19  * OpenShot Library (libopenshot) is distributed in the hope that it will be
20  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22  * GNU Lesser General Public License for more details.
23  *
24  * You should have received a copy of the GNU Lesser General Public License
25  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
26  */
27 
28 #include "../include/FrameMapper.h"
29 
30 using namespace std;
31 using namespace openshot;
32 
33 FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) :
34  reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), timeline_frame_offset(0)
35 {
36  // Set the original frame rate from the reader
37  original = Fraction(reader->info.fps.num, reader->info.fps.den);
38 
39  // Set all info struct members equal to the internal reader
40  info = reader->info;
41  info.fps.num = target.num;
42  info.fps.den = target.den;
43  info.video_timebase.num = target.den;
44  info.video_timebase.den = target.num;
46  info.sample_rate = target_sample_rate;
47  info.channels = target_channels;
48  info.channel_layout = target_channel_layout;
49  info.width = reader->info.width;
50  info.height = reader->info.height;
51 
52  // Used to toggle odd / even fields
53  field_toggle = true;
54 
55  // Adjust cache size based on size of frame and audio
57 
58  // init mapping between original and target frames
59  Init();
60 }
61 
62 // Destructor
64  if (is_open)
65  // Auto Close if not already
66  Close();
67 }
68 
69 /// Get the current reader
71 {
72  if (reader)
73  return reader;
74  else
75  // Throw error if reader not initialized
76  throw ReaderClosed("No Reader has been initialized for FrameMapper. Call Reader(*reader) before calling this method.", "");
77 }
78 
79 void FrameMapper::AddField(int64_t frame)
80 {
81  // Add a field, and toggle the odd / even field
82  AddField(Field(frame, field_toggle));
83 }
84 
85 void FrameMapper::AddField(Field field)
86 {
87  // Add a field to the end of the field list
88  fields.push_back(field);
89 
90  // toggle the odd / even flag
91  field_toggle = (field_toggle ? false : true);
92 }
93 
94 // Use the original and target frame rates and a pull-down technique to create
95 // a mapping between the original fields and frames or a video to a new frame rate.
96 // This might repeat or skip fields and frames of the original video, depending on
97 // whether the frame rate is increasing or decreasing.
98 void FrameMapper::Init()
99 {
100  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Init (Calculate frame mappings)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
101 
102  // Do not initialize anything if just a picture with no audio
104  // Skip initialization
105  return;
106 
107  // Clear the fields & frames lists
108  fields.clear();
109  frames.clear();
110 
111  // Mark as not dirty
112  is_dirty = false;
113 
114  // Clear cache
115  final_cache.Clear();
116 
117  // Some framerates are handled special, and some use a generic Keyframe curve to
118  // map the framerates. These are the special framerates:
119  if ((fabs(original.ToFloat() - 24.0) < 1e-7 || fabs(original.ToFloat() - 25.0) < 1e-7 || fabs(original.ToFloat() - 30.0) < 1e-7) &&
120  (fabs(target.ToFloat() - 24.0) < 1e-7 || fabs(target.ToFloat() - 25.0) < 1e-7 || fabs(target.ToFloat() - 30.0) < 1e-7)) {
121 
122  // Get the difference (in frames) between the original and target frame rates
123  float difference = target.ToInt() - original.ToInt();
124 
125  // Find the number (i.e. interval) of fields that need to be skipped or repeated
126  int field_interval = 0;
127  int frame_interval = 0;
128 
129  if (difference != 0)
130  {
131  field_interval = round(fabs(original.ToInt() / difference));
132 
133  // Get frame interval (2 fields per frame)
134  frame_interval = field_interval * 2.0f;
135  }
136 
137 
138  // Calculate # of fields to map
139  int64_t frame = 1;
140  int64_t number_of_fields = reader->info.video_length * 2;
141 
142  // Loop through all fields in the original video file
143  for (int64_t field = 1; field <= number_of_fields; field++)
144  {
145 
146  if (difference == 0) // Same frame rate, NO pull-down or special techniques required
147  {
148  // Add fields
149  AddField(frame);
150  }
151  else if (difference > 0) // Need to ADD fake fields & frames, because original video has too few frames
152  {
153  // Add current field
154  AddField(frame);
155 
156  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
157  {
158  // Add extra field for each 'field interval
159  AddField(frame);
160  }
161  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
162  {
163  // Add both extra fields in the middle 'together' (i.e. 2:3:3:2 technique)
164  AddField(frame); // add field for current frame
165 
166  if (frame + 1 <= info.video_length)
167  // add field for next frame (if the next frame exists)
168  AddField(Field(frame + 1, field_toggle));
169  }
170  else if (pulldown == PULLDOWN_NONE && field % frame_interval == 0)
171  {
172  // No pull-down technique needed, just repeat this frame
173  AddField(frame);
174  AddField(frame);
175  }
176  }
177  else if (difference < 0) // Need to SKIP fake fields & frames, because we want to return to the original film frame rate
178  {
179 
180  if (pulldown == PULLDOWN_CLASSIC && field % field_interval == 0)
181  {
182  // skip current field and toggle the odd/even flag
183  field_toggle = (field_toggle ? false : true);
184  }
185  else if (pulldown == PULLDOWN_ADVANCED && field % field_interval == 0 && field % frame_interval != 0)
186  {
187  // skip this field, plus the next field
188  field++;
189  }
190  else if (pulldown == PULLDOWN_NONE && frame % field_interval == 0)
191  {
192  // skip this field, plus the next one
193  field++;
194  }
195  else
196  {
197  // No skipping needed, so add the field
198  AddField(frame);
199  }
200  }
201 
202  // increment frame number (if field is divisible by 2)
203  if (field % 2 == 0 && field > 0)
204  frame++;
205  }
206 
207  } else {
208  // Map the remaining framerates using a simple Keyframe curve
209  // Calculate the difference (to be used as a multiplier)
210  double rate_diff = target.ToDouble() / original.ToDouble();
211  int64_t new_length = reader->info.video_length * rate_diff;
212 
213  // Build curve for framerate mapping
214  Keyframe rate_curve;
215  rate_curve.AddPoint(1, 1, LINEAR);
216  rate_curve.AddPoint(new_length, reader->info.video_length, LINEAR);
217 
218  // Loop through curve, and build list of frames
219  for (int64_t frame_num = 1; frame_num <= new_length; frame_num++)
220  {
221  // Add 2 fields per frame
222  AddField(rate_curve.GetInt(frame_num));
223  AddField(rate_curve.GetInt(frame_num));
224  }
225  }
226 
227  // Loop through the target frames again (combining fields into frames)
228  Field Odd(0, true); // temp field used to track the ODD field
229  Field Even(0, true); // temp field used to track the EVEN field
230 
231  // Variables used to remap audio samples
232  int64_t start_samples_frame = 1;
233  int start_samples_position = 0;
234 
235  for (int64_t field = 1; field <= fields.size(); field++)
236  {
237  // Get the current field
238  Field f = fields[field - 1];
239 
240  // Is field divisible by 2?
241  if (field % 2 == 0 && field > 0)
242  {
243  // New frame number
244  int64_t frame_number = field / 2 + timeline_frame_offset;
245 
246  // Set the bottom frame
247  if (f.isOdd)
248  Odd = f;
249  else
250  Even = f;
251 
252  // Determine the range of samples (from the original rate). Resampling happens in real-time when
253  // calling the GetFrame() method. So this method only needs to redistribute the original samples with
254  // the original sample rate.
255  int64_t end_samples_frame = start_samples_frame;
256  int end_samples_position = start_samples_position;
257  int remaining_samples = Frame::GetSamplesPerFrame(frame_number, target, reader->info.sample_rate, reader->info.channels);
258 
259  while (remaining_samples > 0)
260  {
261  // get original samples
262  int original_samples = Frame::GetSamplesPerFrame(end_samples_frame, original, reader->info.sample_rate, reader->info.channels) - end_samples_position;
263 
264  // Enough samples
265  if (original_samples >= remaining_samples)
266  {
267  // Take all that we need, and break loop
268  end_samples_position += remaining_samples - 1;
269  remaining_samples = 0;
270  } else
271  {
272  // Not enough samples (take them all, and keep looping)
273  end_samples_frame += 1; // next frame
274  end_samples_position = 0; // next frame, starting on 1st sample
275  remaining_samples -= original_samples; // reduce the remaining amount
276  }
277  }
278 
279 
280 
281  // Create the sample mapping struct
282  SampleRange Samples = {start_samples_frame, start_samples_position, end_samples_frame, end_samples_position, Frame::GetSamplesPerFrame(frame_number, target, reader->info.sample_rate, reader->info.channels)};
283 
284  // Reset the audio variables
285  start_samples_frame = end_samples_frame;
286  start_samples_position = end_samples_position + 1;
287  if (start_samples_position >= Frame::GetSamplesPerFrame(start_samples_frame, original, reader->info.sample_rate, reader->info.channels))
288  {
289  start_samples_frame += 1; // increment the frame (since we need to wrap onto the next one)
290  start_samples_position = 0; // reset to 0, since we wrapped
291  }
292 
293  // Create a frame and ADD it to the frames collection
294  MappedFrame frame = {Odd, Even, Samples};
295  frames.push_back(frame);
296  }
297  else
298  {
299  // Set the top field
300  if (f.isOdd)
301  Odd = f;
302  else
303  Even = f;
304  }
305  }
306 
307  // Clear the internal fields list (no longer needed)
308  fields.clear();
309 }
310 
311 MappedFrame FrameMapper::GetMappedFrame(int64_t TargetFrameNumber)
312 {
313  // Ignore mapping on single image readers
315  // Return the same number
316  MappedFrame frame;
317  frame.Even.Frame = TargetFrameNumber;
318  frame.Odd.Frame = TargetFrameNumber;
319  frame.Samples.frame_start = 0;
320  frame.Samples.frame_end = 0;
321  frame.Samples.sample_start = 0;
322  frame.Samples.sample_end = 0;
323  frame.Samples.total = 0;
324  return frame;
325  }
326 
327  // Check if frame number is valid
328  if(TargetFrameNumber < 1 || frames.size() == 0)
329  // frame too small, return error
330  throw OutOfBoundsFrame("An invalid frame was requested.", TargetFrameNumber, frames.size());
331 
332  else if (TargetFrameNumber > frames.size())
333  // frame too large, set to end frame
334  TargetFrameNumber = frames.size();
335 
336  // Debug output
337  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetMappedFrame", "TargetFrameNumber", TargetFrameNumber, "frames.size()", frames.size(), "frames[...].Odd", frames[TargetFrameNumber - 1].Odd.Frame, "frames[...].Even", frames[TargetFrameNumber - 1].Even.Frame, "", -1, "", -1);
338 
339  // Return frame
340  return frames[TargetFrameNumber - 1];
341 }
342 
343 // Get or generate a blank frame
344 std::shared_ptr<Frame> FrameMapper::GetOrCreateFrame(int64_t number)
345 {
346  std::shared_ptr<Frame> new_frame;
347 
348  // Init some basic properties about this frame (keep sample rate and # channels the same as the original reader for now)
349  int samples_in_frame = Frame::GetSamplesPerFrame(number + timeline_frame_offset, target, reader->info.sample_rate, reader->info.channels);
350 
351  try {
352  // Debug output
353  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (from reader)", "number", number, "samples_in_frame", samples_in_frame, "", -1, "", -1, "", -1, "", -1);
354 
355  // Set max image size (used for performance optimization)
356  reader->SetMaxSize(max_width, max_height);
357 
358  // Attempt to get a frame (but this could fail if a reader has just been closed)
359  new_frame = reader->GetFrame(number);
360 
361  // Return real frame
362  return new_frame;
363 
364  } catch (const ReaderClosed & e) {
365  // ...
366  } catch (const TooManySeeks & e) {
367  // ...
368  } catch (const OutOfBoundsFrame & e) {
369  // ...
370  }
371 
372  // Debug output
373  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetOrCreateFrame (create blank)", "number", number, "samples_in_frame", samples_in_frame, "", -1, "", -1, "", -1, "", -1);
374 
375  // Create blank frame
376  new_frame = std::make_shared<Frame>(number, info.width, info.height, "#000000", samples_in_frame, reader->info.channels);
377  new_frame->SampleRate(reader->info.sample_rate);
378  new_frame->ChannelsLayout(info.channel_layout);
379  return new_frame;
380 }
381 
382 // Get an openshot::Frame object for a specific frame number of this reader.
383 std::shared_ptr<Frame> FrameMapper::GetFrame(int64_t requested_frame)
384 {
385  // Check final cache, and just return the frame (if it's available)
386  std::shared_ptr<Frame> final_frame = final_cache.GetFrame(requested_frame);
387  if (final_frame) return final_frame;
388 
389  // Create a scoped lock, allowing only a single thread to run the following code at one time
390  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
391 
392  // Check if mappings are dirty (and need to be recalculated)
393  if (is_dirty)
394  // Recalculate mappings
395  Init();
396 
397  // Check final cache a 2nd time (due to potential lock already generating this frame)
398  final_frame = final_cache.GetFrame(requested_frame);
399  if (final_frame) return final_frame;
400 
401  // Minimum number of frames to process (for performance reasons)
402  // Dialing this down to 1 for now, as it seems to improve performance, and reduce export crashes
403  int minimum_frames = 1;
404 
405  // Debug output
406  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (Loop through frames)", "requested_frame", requested_frame, "minimum_frames", minimum_frames, "", -1, "", -1, "", -1, "", -1);
407 
408  // Loop through all requested frames
409  for (int64_t frame_number = requested_frame; frame_number < requested_frame + minimum_frames; frame_number++)
410  {
411 
412  // Debug output
413  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::GetFrame (inside omp for loop)", "frame_number", frame_number, "minimum_frames", minimum_frames, "requested_frame", requested_frame, "", -1, "", -1, "", -1);
414 
415  // Get the mapped frame
416  MappedFrame mapped = GetMappedFrame(frame_number);
417  std::shared_ptr<Frame> mapped_frame;
418 
419  // Get the mapped frame (keeping the sample rate and channels the same as the original... for the moment)
420  mapped_frame = GetOrCreateFrame(mapped.Odd.Frame);
421 
422  // Get # of channels in the actual frame
423  int channels_in_frame = mapped_frame->GetAudioChannelsCount();
424  int samples_in_frame = Frame::GetSamplesPerFrame(frame_number + timeline_frame_offset, target, mapped_frame->SampleRate(), channels_in_frame);
425 
426  // Determine if mapped frame is identical to source frame
427  // including audio sample distribution according to mapped.Samples,
428  // and frame_number. In some cases such as end of stream, the reader
429  // will return a frame with a different frame number. In these cases,
430  // we cannot use the frame as is, nor can we modify the frame number,
431  // otherwise the reader's cache object internals become invalid.
432  if (info.sample_rate == mapped_frame->SampleRate() &&
433  info.channels == mapped_frame->GetAudioChannelsCount() &&
434  info.channel_layout == mapped_frame->ChannelsLayout() &&
435  mapped.Samples.total == mapped_frame->GetAudioSamplesCount() &&
436  mapped.Samples.frame_start == mapped.Odd.Frame &&
437  mapped.Samples.sample_start == 0 &&
438  mapped_frame->number == frame_number &&// in some conditions (e.g. end of stream)
439  info.fps.num == reader->info.fps.num &&
440  info.fps.den == reader->info.fps.den) {
441  // Add original frame to cache, and skip the rest (for performance reasons)
442  final_cache.Add(mapped_frame);
443  continue;
444  }
445 
446  // Create a new frame
447  std::shared_ptr<Frame> frame = std::make_shared<Frame>(frame_number, 1, 1, "#000000", samples_in_frame, channels_in_frame);
448  frame->SampleRate(mapped_frame->SampleRate());
449  frame->ChannelsLayout(mapped_frame->ChannelsLayout());
450 
451 
452  // Copy the image from the odd field
453  std::shared_ptr<Frame> odd_frame;
454  odd_frame = GetOrCreateFrame(mapped.Odd.Frame);
455 
456  if (odd_frame)
457  frame->AddImage(std::shared_ptr<QImage>(new QImage(*odd_frame->GetImage())), true);
458  if (mapped.Odd.Frame != mapped.Even.Frame) {
459  // Add even lines (if different than the previous image)
460  std::shared_ptr<Frame> even_frame;
461  even_frame = GetOrCreateFrame(mapped.Even.Frame);
462  if (even_frame)
463  frame->AddImage(std::shared_ptr<QImage>(new QImage(*even_frame->GetImage())), false);
464  }
465 
466  // Resample audio on frame (if needed)
467  bool need_resampling = false;
468  if (info.has_audio &&
469  (info.sample_rate != frame->SampleRate() ||
470  info.channels != frame->GetAudioChannelsCount() ||
471  info.channel_layout != frame->ChannelsLayout()))
472  // Resample audio and correct # of channels if needed
473  need_resampling = true;
474 
475  // create a copy of mapped.Samples that will be used by copy loop
476  SampleRange copy_samples = mapped.Samples;
477 
478  if (need_resampling)
479  {
480  // Resampling needed, modify copy of SampleRange object that
481  // includes some additional input samples on first iteration,
482  // and continues the offset to ensure that the sample rate
483  // converter isn't input limited.
484  const int EXTRA_INPUT_SAMPLES = 20;
485 
486  // Extend end sample count by an addtional EXTRA_INPUT_SAMPLES samples
487  copy_samples.sample_end += EXTRA_INPUT_SAMPLES;
488  int samples_per_end_frame =
489  Frame::GetSamplesPerFrame(copy_samples.frame_end, original,
490  reader->info.sample_rate, reader->info.channels);
491  if (copy_samples.sample_end >= samples_per_end_frame)
492  {
493  // check for wrapping
494  copy_samples.frame_end++;
495  copy_samples.sample_end -= samples_per_end_frame;
496  }
497  copy_samples.total += EXTRA_INPUT_SAMPLES;
498 
499  if (avr) {
500  // Sample rate conversion has been allocated on this clip, so
501  // this is not the first iteration. Extend start position by
502  // EXTRA_INPUT_SAMPLES to keep step with previous frame
503  copy_samples.sample_start += EXTRA_INPUT_SAMPLES;
504  int samples_per_start_frame =
505  Frame::GetSamplesPerFrame(copy_samples.frame_start, original,
506  reader->info.sample_rate, reader->info.channels);
507  if (copy_samples.sample_start >= samples_per_start_frame)
508  {
509  // check for wrapping
510  copy_samples.frame_start++;
511  copy_samples.sample_start -= samples_per_start_frame;
512  }
513  copy_samples.total -= EXTRA_INPUT_SAMPLES;
514  }
515  }
516 
517  // Copy the samples
518  int samples_copied = 0;
519  int64_t starting_frame = copy_samples.frame_start;
520  while (info.has_audio && samples_copied < copy_samples.total)
521  {
522  // Init number of samples to copy this iteration
523  int remaining_samples = copy_samples.total - samples_copied;
524  int number_to_copy = 0;
525 
526  // number of original samples on this frame
527  std::shared_ptr<Frame> original_frame = GetOrCreateFrame(starting_frame);
528  int original_samples = original_frame->GetAudioSamplesCount();
529 
530  // Loop through each channel
531  for (int channel = 0; channel < channels_in_frame; channel++)
532  {
533  if (starting_frame == copy_samples.frame_start)
534  {
535  // Starting frame (take the ending samples)
536  number_to_copy = original_samples - copy_samples.sample_start;
537  if (number_to_copy > remaining_samples)
538  number_to_copy = remaining_samples;
539 
540  // Add samples to new frame
541  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.sample_start, number_to_copy, 1.0);
542  }
543  else if (starting_frame > copy_samples.frame_start && starting_frame < copy_samples.frame_end)
544  {
545  // Middle frame (take all samples)
546  number_to_copy = original_samples;
547  if (number_to_copy > remaining_samples)
548  number_to_copy = remaining_samples;
549 
550  // Add samples to new frame
551  frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
552  }
553  else
554  {
555  // Ending frame (take the beginning samples)
556  number_to_copy = copy_samples.sample_end + 1;
557  if (number_to_copy > remaining_samples)
558  number_to_copy = remaining_samples;
559 
560  // Add samples to new frame
561  frame->AddAudio(false, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
562  }
563  }
564 
565  // increment frame
566  samples_copied += number_to_copy;
567  starting_frame++;
568  }
569 
570  // Resample audio on frame (if needed)
571  if (need_resampling)
572  // Resample audio and correct # of channels if needed
573  ResampleMappedAudio(frame, mapped.Odd.Frame);
574 
575  // Add frame to final cache
576  final_cache.Add(frame);
577 
578  } // for loop
579 
580  // Return processed openshot::Frame
581  return final_cache.GetFrame(requested_frame);
582 }
583 
585 {
586  // Check if mappings are dirty (and need to be recalculated)
587  if (is_dirty)
588  // Recalculate mappings
589  Init();
590 
591  // Get the difference (in frames) between the original and target frame rates
592  float difference = target.ToInt() - original.ToInt();
593 
594  int field_interval = 0;
595  int frame_interval = 0;
596 
597  if (difference != 0)
598  {
599  // Find the number (i.e. interval) of fields that need to be skipped or repeated
600  field_interval = round(fabs(original.ToInt() / difference));
601 
602  // Get frame interval (2 fields per frame)
603  frame_interval = field_interval * 2.0f;
604  }
605 
606  // Loop through frame mappings
607  for (float map = 1; map <= frames.size(); map++)
608  {
609  MappedFrame frame = frames[map - 1];
610  cout << "Target frame #: " << map << " mapped to original frame #:\t(" << frame.Odd.Frame << " odd, " << frame.Even.Frame << " even)" << endl;
611  cout << " - Audio samples mapped to frame " << frame.Samples.frame_start << ":" << frame.Samples.sample_start << " to frame " << frame.Samples.frame_end << ":" << frame.Samples.sample_end << endl;
612  }
613 
614 }
615 
616 
617 // Determine if reader is open or closed
619  if (reader)
620  return reader->IsOpen();
621  else
622  return false;
623 }
624 
625 
626 // Open the internal reader
628 {
629  if (reader)
630  {
631  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Open", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
632 
633  // Open the reader
634  reader->Open();
635  }
636 }
637 
638 // Close the internal reader
640 {
641  if (reader)
642  {
643  // Create a scoped lock, allowing only a single thread to run the following code at one time
644  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
645 
646  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
647 
648  // Close internal reader
649  reader->Close();
650 
651  // Deallocate resample buffer
652  if (avr) {
653  avresample_close(avr);
654  avresample_free(&avr);
655  avr = NULL;
656  }
657  }
658 }
659 
660 
661 // Generate JSON string of this object
663 
664  // Return formatted string
665  return JsonValue().toStyledString();
666 }
667 
668 // Generate Json::JsonValue for this object
669 Json::Value FrameMapper::JsonValue() {
670 
671  // Create root json object
672  Json::Value root = ReaderBase::JsonValue(); // get parent properties
673  root["type"] = "FrameMapper";
674 
675  // return JsonValue
676  return root;
677 }
678 
679 // Load JSON string into this object
680 void FrameMapper::SetJson(string value) {
681 
682  // Parse JSON string into JSON objects
683  Json::Value root;
684  Json::Reader reader;
685  bool success = reader.parse( value, root );
686  if (!success)
687  // Raise exception
688  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
689 
690  try
691  {
692  // Set all values that match
693  SetJsonValue(root);
694  }
695  catch (exception e)
696  {
697  // Error parsing JSON (or missing keys)
698  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
699  }
700 }
701 
702 // Load Json::JsonValue into this object
703 void FrameMapper::SetJsonValue(Json::Value root) {
704 
705  // Set parent data
707 
708  // Re-Open path, and re-init everything (if needed)
709  if (reader) {
710 
711  Close();
712  Open();
713  }
714 }
715 
716 // Change frame rate or audio mapping details
717 void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
718 {
719  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ChangeMapping", "target_fps.num", target_fps.num, "target_fps.den", target_fps.den, "target_pulldown", target_pulldown, "target_sample_rate", target_sample_rate, "target_channels", target_channels, "target_channel_layout", target_channel_layout);
720 
721  // Mark as dirty
722  is_dirty = true;
723 
724  // Update mapping details
725  target.num = target_fps.num;
726  target.den = target_fps.den;
727  info.fps.num = target_fps.num;
728  info.fps.den = target_fps.den;
729  info.video_timebase.num = target_fps.den;
730  info.video_timebase.den = target_fps.num;
731  pulldown = target_pulldown;
732  info.sample_rate = target_sample_rate;
733  info.channels = target_channels;
734  info.channel_layout = target_channel_layout;
735 
736  // Clear cache
737  final_cache.Clear();
738 
739  // Adjust cache size based on size of frame and audio
741 
742  // Deallocate resample buffer
743  if (avr) {
744  avresample_close(avr);
745  avresample_free(&avr);
746  avr = NULL;
747  }
748 
749  // Re-init mapping
750  Init();
751 }
752 
753 // Set offset relative to parent timeline
755 {
756  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::SetTimelineFrameOffset", "offset", offset, "", -1, "", -1, "", -1, "", -1, "", -1);
757 
758  // Mark as dirty
759  is_dirty = true;
760 
761  // Used to correctly align audio sample distribution
762  timeline_frame_offset = offset;
763 }
764 
765 // Resample audio and map channels (if needed)
766 void FrameMapper::ResampleMappedAudio(std::shared_ptr<Frame> frame, int64_t original_frame_number)
767 {
768  // Init audio buffers / variables
769  int total_frame_samples = 0;
770  int channels_in_frame = frame->GetAudioChannelsCount();
771  int sample_rate_in_frame = frame->SampleRate();
772  int samples_in_frame = frame->GetAudioSamplesCount();
773  ChannelLayout channel_layout_in_frame = frame->ChannelsLayout();
774 
775  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio", "frame->number", frame->number, "original_frame_number", original_frame_number, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "sample_rate_in_frame", sample_rate_in_frame, "", -1);
776 
777  // Get audio sample array
778  float* frame_samples_float = NULL;
779  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
780  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
781 
782  // Calculate total samples
783  total_frame_samples = samples_in_frame * channels_in_frame;
784 
785  // Create a new array (to hold all S16 audio samples for the current queued frames)
786  int16_t* frame_samples = (int16_t*) av_malloc(sizeof(int16_t)*total_frame_samples);
787 
788  // Translate audio sample values back to 16 bit integers
789  for (int s = 0; s < total_frame_samples; s++)
790  // Translate sample value and copy into buffer
791  frame_samples[s] = int(frame_samples_float[s] * (1 << 15));
792 
793 
794  // Deallocate float array
795  delete[] frame_samples_float;
796  frame_samples_float = NULL;
797 
798  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (got sample data from frame)", "frame->number", frame->number, "total_frame_samples", total_frame_samples, "target channels", info.channels, "channels_in_frame", channels_in_frame, "target sample_rate", info.sample_rate, "samples_in_frame", samples_in_frame);
799 
800 
801  // Create input frame (and allocate arrays)
802  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
803  AV_RESET_FRAME(audio_frame);
804  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
805 
806  int error_code = avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) frame_samples,
807  audio_frame->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * channels_in_frame, 1);
808 
809  if (error_code < 0)
810  {
811  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
812  throw ErrorEncodingVideo("Error while resampling audio in frame mapper", frame->number);
813  }
814 
815  // Update total samples & input frame size (due to bigger or smaller data types)
816  total_frame_samples = Frame::GetSamplesPerFrame(frame->number + timeline_frame_offset, target, info.sample_rate, info.channels);
817 
818  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (adjust # of samples)", "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "sample_rate_in_frame", sample_rate_in_frame, "info.channels", info.channels, "channels_in_frame", channels_in_frame, "original_frame_number", original_frame_number);
819 
820  // Create output frame (and allocate arrays)
821  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
822  AV_RESET_FRAME(audio_converted);
823  audio_converted->nb_samples = total_frame_samples;
824  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, total_frame_samples, AV_SAMPLE_FMT_S16, 0);
825 
826  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (preparing for resample)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", AV_SAMPLE_FMT_S16, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
827 
828  int nb_samples = 0;
829 
830  // setup resample context
831  if (!avr) {
832  avr = avresample_alloc_context();
833  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
834  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
835  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
836  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
837  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
838  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
839  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
840  av_opt_set_int(avr, "out_channels", info.channels, 0);
841  avresample_open(avr);
842  }
843 
844  // Convert audio samples
845  nb_samples = avresample_convert(avr, // audio resample context
846  audio_converted->data, // output data pointers
847  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
848  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
849  audio_frame->data, // input data pointers
850  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
851  audio_frame->nb_samples); // number of input samples to convert
852 
853  // Create a new array (to hold all resampled S16 audio samples)
854  int16_t* resampled_samples = new int16_t[(nb_samples * info.channels)];
855 
856  // Copy audio samples over original samples
857  memcpy(resampled_samples, audio_converted->data[0], (nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels));
858 
859  // Free frames
860  av_freep(&audio_frame->data[0]);
861  AV_FREE_FRAME(&audio_frame);
862  av_freep(&audio_converted->data[0]);
863  AV_FREE_FRAME(&audio_converted);
864  frame_samples = NULL;
865 
866  // Resize the frame to hold the right # of channels and samples
867  int channel_buffer_size = nb_samples;
868  frame->ResizeAudio(info.channels, channel_buffer_size, info.sample_rate, info.channel_layout);
869 
870  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Audio successfully resampled)", "nb_samples", nb_samples, "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "channels_in_frame", channels_in_frame, "info.channels", info.channels, "info.channel_layout", info.channel_layout);
871 
872  // Array of floats (to hold samples for each channel)
873  float *channel_buffer = new float[channel_buffer_size];
874 
875  // Divide audio into channels. Loop through each channel
876  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
877  {
878  // Init array
879  for (int z = 0; z < channel_buffer_size; z++)
880  channel_buffer[z] = 0.0f;
881 
882  // Loop through all samples and add them to our Frame based on channel.
883  // Toggle through each channel number, since channel data is stored like (left right left right)
884  int channel = 0;
885  int position = 0;
886  for (int sample = 0; sample < (nb_samples * info.channels); sample++)
887  {
888  // Only add samples for current channel
889  if (channel_filter == channel)
890  {
891  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
892  channel_buffer[position] = resampled_samples[sample] * (1.0f / (1 << 15));
893 
894  // Increment audio position
895  position++;
896  }
897 
898  // increment channel (if needed)
899  if ((channel + 1) < info.channels)
900  // move to next channel
901  channel ++;
902  else
903  // reset channel
904  channel = 0;
905  }
906 
907  // Add samples to frame for this channel
908  frame->AddAudio(true, channel_filter, 0, channel_buffer, position, 1.0f);
909 
910  ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (Add audio to channel)", "number of samples", position, "channel_filter", channel_filter, "", -1, "", -1, "", -1, "", -1);
911  }
912 
913  // Update frame's audio meta data
914  frame->SampleRate(info.sample_rate);
915  frame->ChannelsLayout(info.channel_layout);
916 
917  // clear channel buffer
918  delete[] channel_buffer;
919  channel_buffer = NULL;
920 
921  // Delete arrays
922  delete[] resampled_samples;
923  resampled_samples = NULL;
924 }
#define AV_RESET_FRAME(av_frame)
Classic 2:3:2:3 pull-down.
Definition: FrameMapper.h:62
int max_height
The maximium image height needed by this clip (used for optimizations)
Definition: ReaderBase.h:103
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:44
ReaderBase * Reader()
Get the current reader.
Definition: FrameMapper.cpp:70
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
Definition: ReaderBase.h:99
vector< MappedFrame > frames
Definition: FrameMapper.h:168
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:83
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:67
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
~FrameMapper()
Destructor.
Definition: FrameMapper.cpp:63
float duration
Length of time (in seconds)
Definition: ReaderBase.h:64
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:46
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
void Close()
Close the openshot::FrameMapper and internal reader.
virtual void Close()=0
Close the reader (and any resources it was consuming)
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:95
#define OPEN_MP_NUM_PROCESSORS
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:61
This struct holds a single field (half a frame).
Definition: FrameMapper.h:73
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
This method is required for all derived classes of ReaderBase, and return the openshot::Frame object...
Exception when encoding audio packet.
Definition: Exceptions.h:101
void AddPoint(Point p)
Add a new point on the key-frame. Each point has a primary coordinate, a left handle, and a right handle.
Definition: KeyFrame.cpp:72
This struct holds a the range of samples needed by this frame.
Definition: FrameMapper.h:93
virtual std::shared_ptr< Frame > GetFrame(int64_t number)=0
void SetMaxSize(int width, int height)
Set Max Image Size (used for performance optimization)
Definition: ReaderBase.h:143
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:62
void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout)
Change frame rate or audio mapping details.
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:74
int height
The height of the video (in pixels)
Definition: ReaderBase.h:66
#define AV_ALLOCATE_FRAME()
bool IsOpen()
Determine if reader is open or closed.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
bool has_single_image
Determines if this file only contains a single image.
Definition: ReaderBase.h:63
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:106
int GetInt(int64_t index)
Get the rounded INT value at a specific index.
Definition: KeyFrame.cpp:248
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:155
#define av_err2str(errnum)
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:51
This struct holds two fields which together make up a complete video frame.
Definition: FrameMapper.h:110
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:69
vector< Field > fields
Definition: FrameMapper.h:167
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:76
string Json()
Get and Set JSON methods.
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
This namespace is the default namespace for all code in the openshot library.
Do not apply pull-down techniques, just repeat or skip entire frames.
Definition: FrameMapper.h:64
Linear curves are angular, straight lines between two points.
Definition: Point.h:47
void PrintMapping()
Print all of the original frames and which new frames they map to.
Exception for invalid JSON.
Definition: Exceptions.h:152
void ResampleMappedAudio(std::shared_ptr< Frame > frame, int64_t original_frame_number)
Resample audio and map channels (if needed)
MappedFrame GetMappedFrame(int64_t TargetFrameNumber)
Get a frame based on the target frame rate and the new frame number of a frame.
PulldownType
This enumeration determines how frame rates are increased or decreased.
Definition: FrameMapper.h:60
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:82
A Keyframe is a collection of Point instances, which is used to vary a number or property over time...
Definition: KeyFrame.h:64
int max_width
The maximum image width needed by this clip (used for optimizations)
Definition: ReaderBase.h:102
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:505
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Advanced 2:3:3:2 pull-down (minimal dirty frames)
Definition: FrameMapper.h:63
void SetJson(string value)
Load JSON string into this object.
void Open()
Open the internal reader.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
virtual void Open()=0
Open the reader (and start consuming resources, such as images or video files)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:81
Exception when too many seek attempts happen.
Definition: Exceptions.h:254
virtual bool IsOpen()=0
Determine if reader is open or closed.
void SetTimelineFrameOffset(int64_t offset)