OpenShot Library | libopenshot  0.1.9
FFmpegReader.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegReader class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegReader.h"
32 
33 using namespace openshot;
34 
36  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), packet(NULL) {
41 
42  // Initialize FFMpeg, and register all formats and codecs
43  av_register_all();
44  avcodec_register_all();
45 
46  // Init cache
50 
51  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
52  Open();
53  Close();
54 }
55 
56 FFmpegReader::FFmpegReader(string path, bool inspect_reader)
57  : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
58  audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
59  check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
60  prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
61  current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), packet(NULL) {
62 
63  // Initialize FFMpeg, and register all formats and codecs
64  av_register_all();
65  avcodec_register_all();
66 
67  // Init cache
71 
72  // Open and Close the reader, to populate it's attributes (such as height, width, etc...)
73  if (inspect_reader) {
74  Open();
75  Close();
76  }
77 }
78 
80  if (is_open)
81  // Auto close reader if not already done
82  Close();
83 }
84 
85 // This struct holds the associated video frame and starting sample # for an audio packet.
86 bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount)
87 {
88  // Is frame even close to this one?
89  if (abs(location.frame - frame) >= 2)
90  // This is too far away to be considered
91  return false;
92 
93  // Note that samples_per_frame can vary slightly frame to frame when the
94  // audio sampling rate is not an integer multiple of the video fps.
95  int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
96  if (abs(diff) <= amount)
97  // close
98  return true;
99 
100  // not close
101  return false;
102 }
103 
105 {
106  // Open reader if not already open
107  if (!is_open)
108  {
109  // Initialize format context
110  pFormatCtx = NULL;
111 
112  // Open video file
113  if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
114  throw InvalidFile("File could not be opened.", path);
115 
116  // Retrieve stream information
117  if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
118  throw NoStreamsFound("No streams found in file.", path);
119 
120  videoStream = -1;
121  audioStream = -1;
122  // Loop through each stream, and identify the video and audio stream index
123  for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
124  {
125  // Is this a video stream?
126  if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
127  videoStream = i;
128  }
129  // Is this an audio stream?
130  if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
131  audioStream = i;
132  }
133  }
134  if (videoStream == -1 && audioStream == -1)
135  throw NoStreamsFound("No video or audio streams found in this file.", path);
136 
137  // Is there a video stream?
138  if (videoStream != -1)
139  {
140  // Set the stream index
141  info.video_stream_index = videoStream;
142 
143  // Set the codec and codec context pointers
144  pStream = pFormatCtx->streams[videoStream];
145  pCodecCtx = pFormatCtx->streams[videoStream]->codec;
146 
147  // Set number of threads equal to number of processors (not to exceed 16)
148  pCodecCtx->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
149 
150  // Find the decoder for the video stream
151  AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
152  if (pCodec == NULL) {
153  throw InvalidCodec("A valid video codec could not be found for this file.", path);
154  }
155  // Open video codec
156  if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
157  throw InvalidCodec("A video codec was found, but could not be opened.", path);
158 
159  // Update the File Info struct with video details (if a video stream is found)
160  UpdateVideoInfo();
161  }
162 
163  // Is there an audio stream?
164  if (audioStream != -1)
165  {
166  // Set the stream index
167  info.audio_stream_index = audioStream;
168 
169  // Get a pointer to the codec context for the audio stream
170  aStream = pFormatCtx->streams[audioStream];
171  aCodecCtx = pFormatCtx->streams[audioStream]->codec;
172 
173  // Set number of threads equal to number of processors (not to exceed 16)
174  aCodecCtx->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
175 
176  // Find the decoder for the audio stream
177  AVCodec *aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
178  if (aCodec == NULL) {
179  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
180  }
181  // Open audio codec
182  if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0)
183  throw InvalidCodec("An audio codec was found, but could not be opened.", path);
184 
185  // Update the File Info struct with audio details (if an audio stream is found)
186  UpdateAudioInfo();
187  }
188 
189  // Init previous audio location to zero
190  previous_packet_location.frame = -1;
191  previous_packet_location.sample_start = 0;
192 
193  // Adjust cache size based on size of frame and audio
197 
198  // Mark as "open"
199  is_open = true;
200  }
201 }
202 
204 {
205  // Close all objects, if reader is 'open'
206  if (is_open)
207  {
208  // Mark as "closed"
209  is_open = false;
210 
211  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
212 
213  // Close the codec
214  if (info.has_video)
215  {
216  avcodec_flush_buffers(pCodecCtx);
217  avcodec_close(pCodecCtx);
218  }
219  if (info.has_audio)
220  {
221  avcodec_flush_buffers(aCodecCtx);
222  avcodec_close(aCodecCtx);
223  }
224 
225  // Clear final cache
226  final_cache.Clear();
227  working_cache.Clear();
228  missing_frames.Clear();
229 
230  // Clear processed lists
231  {
232  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
233  processed_video_frames.clear();
234  processed_audio_frames.clear();
235  processing_video_frames.clear();
236  processing_audio_frames.clear();
237  missing_audio_frames.clear();
238  missing_video_frames.clear();
239  missing_audio_frames_source.clear();
240  missing_video_frames_source.clear();
241  checked_frames.clear();
242  }
243 
244  // Close the video file
245  avformat_close_input(&pFormatCtx);
246  av_freep(&pFormatCtx);
247 
248  // Reset some variables
249  last_frame = 0;
250  largest_frame_processed = 0;
251  seek_audio_frame_found = 0;
252  seek_video_frame_found = 0;
253  current_video_frame = 0;
254  has_missing_frames = false;
255  }
256 }
257 
258 void FFmpegReader::UpdateAudioInfo()
259 {
260  // Set values of FileInfo struct
261  info.has_audio = true;
262  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
263  info.acodec = aCodecCtx->codec->name;
264  info.channels = aCodecCtx->channels;
265  if (aCodecCtx->channel_layout == 0)
266  aCodecCtx->channel_layout = av_get_default_channel_layout( aCodecCtx->channels );
267  info.channel_layout = (ChannelLayout) aCodecCtx->channel_layout;
268  info.sample_rate = aCodecCtx->sample_rate;
269  info.audio_bit_rate = aCodecCtx->bit_rate;
270 
271  // Set audio timebase
272  info.audio_timebase.num = aStream->time_base.num;
273  info.audio_timebase.den = aStream->time_base.den;
274 
275  // Get timebase of audio stream (if valid) and greater than the current duration
276  if (aStream->duration > 0.0f && aStream->duration > info.duration)
277  info.duration = aStream->duration * info.audio_timebase.ToDouble();
278 
279  // Check for an invalid video length
280  if (info.has_video && info.video_length <= 0)
281  {
282  // Calculate the video length from the audio duration
284  }
285 
286  // Set video timebase (if no video stream was found)
287  if (!info.has_video)
288  {
289  // Set a few important default video settings (so audio can be divided into frames)
290  info.fps.num = 24;
291  info.fps.den = 1;
292  info.video_timebase.num = 1;
293  info.video_timebase.den = 24;
295  info.width = 720;
296  info.height = 480;
297 
298  }
299 
300 }
301 
302 void FFmpegReader::UpdateVideoInfo()
303 {
304  // Set values of FileInfo struct
305  info.has_video = true;
306  info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
307  info.height = pCodecCtx->height;
308  info.width = pCodecCtx->width;
309  info.vcodec = pCodecCtx->codec->name;
310  info.video_bit_rate = pFormatCtx->bit_rate;
311  if (!check_fps)
312  {
313  // set frames per second (fps)
314  info.fps.num = pStream->avg_frame_rate.num;
315  info.fps.den = pStream->avg_frame_rate.den;
316  }
317 
318  if (pStream->sample_aspect_ratio.num != 0)
319  {
320  info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
321  info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
322  }
323  else if (pCodecCtx->sample_aspect_ratio.num != 0)
324  {
325  info.pixel_ratio.num = pCodecCtx->sample_aspect_ratio.num;
326  info.pixel_ratio.den = pCodecCtx->sample_aspect_ratio.den;
327  }
328  else
329  {
330  info.pixel_ratio.num = 1;
331  info.pixel_ratio.den = 1;
332  }
333 
334  info.pixel_format = pCodecCtx->pix_fmt;
335 
336  // Calculate the DAR (display aspect ratio)
338 
339  // Reduce size fraction
340  size.Reduce();
341 
342  // Set the ratio based on the reduced fraction
343  info.display_ratio.num = size.num;
344  info.display_ratio.den = size.den;
345 
346  // Set the video timebase
347  info.video_timebase.num = pStream->time_base.num;
348  info.video_timebase.den = pStream->time_base.den;
349 
350  // Set the duration in seconds, and video length (# of frames)
351  info.duration = pStream->duration * info.video_timebase.ToDouble();
352 
353  // Check for valid duration (if found)
354  if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
355  // Use the format's duration
356  info.duration = pFormatCtx->duration / AV_TIME_BASE;
357 
358  // Calculate duration from filesize and bitrate (if any)
359  if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
360  // Estimate from bitrate, total bytes, and framerate
362 
363  // No duration found in stream of file
364  if (info.duration <= 0.0f)
365  {
366  // No duration is found in the video stream
367  info.duration = -1;
368  info.video_length = -1;
369  is_duration_known = false;
370  }
371  else
372  {
373  // Yes, a duration was found
374  is_duration_known = true;
375 
376  // Calculate number of frames
378  }
379 
380  // Override an invalid framerate
381  if (info.fps.ToFloat() > 120.0f || (info.fps.num == 0 || info.fps.den == 0))
382  {
383  // Set a few important default video settings (so audio can be divided into frames)
384  info.fps.num = 24;
385  info.fps.den = 1;
386  info.video_timebase.num = 1;
387  info.video_timebase.den = 24;
388 
389  // Calculate number of frames
391  }
392 
393 }
394 
395 
396 std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame)
397 {
398  // Check for open reader (or throw exception)
399  if (!is_open)
400  throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
401 
402  // Adjust for a requested frame that is too small or too large
403  if (requested_frame < 1)
404  requested_frame = 1;
405  if (requested_frame > info.video_length && is_duration_known)
406  requested_frame = info.video_length;
407  if (info.has_video && info.video_length == 0)
408  // Invalid duration of video file
409  throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
410 
411  // Debug output
412  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame, "", -1, "", -1, "", -1, "", -1);
413 
414  // Check the cache for this frame
415  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
416  if (frame) {
417  // Debug output
418  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
419 
420  // Return the cached frame
421  return frame;
422  }
423  else
424  {
425  // Create a scoped lock, allowing only a single thread to run the following code at one time
426  const GenericScopedLock<CriticalSection> lock(getFrameCriticalSection);
427 
428  // Check the cache a 2nd time (due to a potential previous lock)
429  if (has_missing_frames)
430  CheckMissingFrame(requested_frame);
431  frame = final_cache.GetFrame(requested_frame);
432  if (frame) {
433  // Debug output
434  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1);
435 
436  // Return the cached frame
437  return frame;
438  }
439 
440  // Frame is not in cache
441  // Reset seek count
442  seek_count = 0;
443 
444  // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
445  if (last_frame == 0 && requested_frame != 1)
446  // Get first frame
447  ReadStream(1);
448 
449  // Are we within X frames of the requested frame?
450  int64_t diff = requested_frame - last_frame;
451  if (diff >= 1 && diff <= 20)
452  {
453  // Continue walking the stream
454  return ReadStream(requested_frame);
455  }
456  else
457  {
458  // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
459  if (enable_seek)
460  // Only seek if enabled
461  Seek(requested_frame);
462 
463  else if (!enable_seek && diff < 0)
464  {
465  // Start over, since we can't seek, and the requested frame is smaller than our position
466  Close();
467  Open();
468  }
469 
470  // Then continue walking the stream
471  return ReadStream(requested_frame);
472  }
473 
474  }
475 }
476 
477 // Read the stream until we find the requested Frame
478 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
479 {
480  // Allocate video frame
481  bool end_of_stream = false;
482  bool check_seek = false;
483  bool frame_finished = false;
484  int packet_error = -1;
485 
486  // Minimum number of packets to process (for performance reasons)
487  int packets_processed = 0;
488  int minimum_packets = OPEN_MP_NUM_PROCESSORS;
489  int max_packets = 4096;
490 
491  // Set the number of threads in OpenMP
492  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
493  // Allow nested OpenMP sections
494  omp_set_nested(true);
495 
496  // Debug output
497  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "OPEN_MP_NUM_PROCESSORS", OPEN_MP_NUM_PROCESSORS, "", -1, "", -1, "", -1, "", -1);
498 
499  #pragma omp parallel
500  {
501  #pragma omp single
502  {
503  // Loop through the stream until the correct frame is found
504  while (true)
505  {
506  // Get the next packet into a local variable called packet
507  packet_error = GetNextPacket();
508 
509  int processing_video_frames_size = 0;
510  int processing_audio_frames_size = 0;
511  {
512  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
513  processing_video_frames_size = processing_video_frames.size();
514  processing_audio_frames_size = processing_audio_frames.size();
515  }
516 
517  // Wait if too many frames are being processed
518  while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
519  usleep(2500);
520  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
521  processing_video_frames_size = processing_video_frames.size();
522  processing_audio_frames_size = processing_audio_frames.size();
523  }
524 
525  // Get the next packet (if any)
526  if (packet_error < 0)
527  {
528  // Break loop when no more packets found
529  end_of_stream = true;
530  break;
531  }
532 
533  // Debug output
534  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking);
535 
536  // Video packet
537  if (info.has_video && packet->stream_index == videoStream)
538  {
539  // Reset this counter, since we have a video packet
540  num_packets_since_video_frame = 0;
541 
542  // Check the status of a seek (if any)
543  if (is_seeking)
544  #pragma omp critical (openshot_seek)
545  check_seek = CheckSeek(true);
546  else
547  check_seek = false;
548 
549  if (check_seek) {
550  // Jump to the next iteration of this loop
551  continue;
552  }
553 
554  // Get the AVFrame from the current packet
555  frame_finished = GetAVFrame();
556 
557  // Check if the AVFrame is finished and set it
558  if (frame_finished)
559  {
560  // Update PTS / Frame Offset (if any)
561  UpdatePTSOffset(true);
562 
563  // Process Video Packet
564  ProcessVideoPacket(requested_frame);
565  }
566 
567  }
568  // Audio packet
569  else if (info.has_audio && packet->stream_index == audioStream)
570  {
571  // Increment this (to track # of packets since the last video packet)
572  num_packets_since_video_frame++;
573 
574  // Check the status of a seek (if any)
575  if (is_seeking)
576  #pragma omp critical (openshot_seek)
577  check_seek = CheckSeek(false);
578  else
579  check_seek = false;
580 
581  if (check_seek) {
582  // Jump to the next iteration of this loop
583  continue;
584  }
585 
586  // Update PTS / Frame Offset (if any)
587  UpdatePTSOffset(false);
588 
589  // Determine related video frame and starting sample # from audio PTS
590  AudioLocation location = GetAudioPTSLocation(packet->pts);
591 
592  // Process Audio Packet
593  ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
594  }
595 
596  // Check if working frames are 'finished'
597  bool is_cache_found = false;
598  if (!is_seeking) {
599  // Check for any missing frames
600  CheckMissingFrame(requested_frame);
601 
602  // Check for final frames
603  CheckWorkingFrames(false, requested_frame);
604  }
605 
606  // Check if requested 'final' frame is available
607  is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
608 
609  // Increment frames processed
610  packets_processed++;
611 
612  // Break once the frame is found
613  if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
614  break;
615 
616  } // end while
617 
618  } // end omp single
619  } // end omp parallel
620 
621  // Debug output
622  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count(), "", -1, "", -1);
623 
624  // End of stream?
625  if (end_of_stream)
626  // Mark the any other working frames as 'finished'
627  CheckWorkingFrames(end_of_stream, requested_frame);
628 
629  // Return requested frame (if found)
630  std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
631  if (frame)
632  // Return prepared frame
633  return frame;
634  else {
635 
636  // Check if largest frame is still cached
637  frame = final_cache.GetFrame(largest_frame_processed);
638  if (frame) {
639  // return the largest processed frame (assuming it was the last in the video file)
640  return frame;
641  }
642  else {
643  // The largest processed frame is no longer in cache, return a blank frame
644  std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
645  f->AddColor(info.width, info.height, "#000");
646  return f;
647  }
648  }
649 
650 }
651 
652 // Get the next packet (if any)
653 int FFmpegReader::GetNextPacket()
654 {
655  int found_packet = 0;
656  AVPacket *next_packet = new AVPacket();
657  found_packet = av_read_frame(pFormatCtx, next_packet);
658 
659  if (packet) {
660  // Remove previous packet before getting next one
661  RemoveAVPacket(packet);
662  packet = NULL;
663  }
664 
665  if (found_packet >= 0)
666  {
667  // Update current packet pointer
668  packet = next_packet;
669  }
670 
671  // Return if packet was found (or error number)
672  return found_packet;
673 }
674 
675 // Get an AVFrame (if any)
676 bool FFmpegReader::GetAVFrame()
677 {
678  int frameFinished = -1;
679 
680  // Decode video frame
681  AVFrame *next_frame = AV_ALLOCATE_FRAME();
682  #pragma omp critical (packet_cache)
683  avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
684 
685  // is frame finished
686  if (frameFinished)
687  {
688  // AVFrames are clobbered on the each call to avcodec_decode_video, so we
689  // must make a copy of the image data before this method is called again.
690  pFrame = new AVPicture();
691  avpicture_alloc(pFrame, pCodecCtx->pix_fmt, info.width, info.height);
692  av_picture_copy(pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width, info.height);
693 
694  // Detect interlaced frame (only once)
695  if (!check_interlace)
696  {
697  check_interlace = true;
698  info.interlaced_frame = next_frame->interlaced_frame;
699  info.top_field_first = next_frame->top_field_first;
700  }
701  }
702 
703  // deallocate the frame
704  AV_FREE_FRAME(&next_frame);
705 
706  // Did we get a video frame?
707  return frameFinished;
708 }
709 
710 // Check the current seek position and determine if we need to seek again
711 bool FFmpegReader::CheckSeek(bool is_video)
712 {
713  // Are we seeking for a specific frame?
714  if (is_seeking)
715  {
716  // Determine if both an audio and video packet have been decoded since the seek happened.
717  // If not, allow the ReadStream method to keep looping
718  if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
719  return false;
720 
721  // Check for both streams
722  if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
723  return false;
724 
725  // Determine max seeked frame
726  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
727  if (seek_video_frame_found > max_seeked_frame)
728  max_seeked_frame = seek_video_frame_found;
729 
730  // determine if we are "before" the requested frame
731  if (max_seeked_frame >= seeking_frame)
732  {
733  // SEEKED TOO FAR
734  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
735 
736  // Seek again... to the nearest Keyframe
737  Seek(seeking_frame - (20 * seek_count * seek_count));
738  }
739  else
740  {
741  // SEEK WORKED
742  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
743 
744  // Seek worked, and we are "before" the requested frame
745  is_seeking = false;
746  seeking_frame = 0;
747  seeking_pts = -1;
748  }
749  }
750 
751  // return the pts to seek to (if any)
752  return is_seeking;
753 }
754 
755 // Process a video packet
756 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame)
757 {
758  // Calculate current frame #
759  int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
760 
761  // Track 1st video packet after a successful seek
762  if (!seek_video_frame_found && is_seeking)
763  seek_video_frame_found = current_frame;
764 
765  // Are we close enough to decode the frame? and is this frame # valid?
766  if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
767  {
768  // Remove frame and packet
769  RemoveAVFrame(pFrame);
770 
771  // Debug output
772  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
773 
774  // Skip to next frame without decoding or caching
775  return;
776  }
777 
778  // Debug output
779  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame, "", -1, "", -1, "", -1, "", -1);
780 
781  // Init some things local (for OpenMP)
782  PixelFormat pix_fmt = pCodecCtx->pix_fmt;
783  int height = info.height;
784  int width = info.width;
785  int64_t video_length = info.video_length;
786  AVPicture *my_frame = pFrame;
787 
788  // Add video frame to list of processing video frames
789  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
790  processing_video_frames[current_frame] = current_frame;
791 
792  #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt)
793  {
794  // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
795  AVFrame *pFrameRGB = NULL;
796  int numBytes;
797  uint8_t *buffer = NULL;
798 
799  // Allocate an AVFrame structure
800  pFrameRGB = AV_ALLOCATE_FRAME();
801  if (pFrameRGB == NULL)
802  throw OutOfBoundsFrame("Convert Image Broke!", current_frame, video_length);
803 
804  // Determine if video needs to be scaled down (for performance reasons)
805  // Timelines pass their size to the clips, which pass their size to the readers (as max size)
806  // If a clip is being scaled larger, it will set max_width and max_height = 0 (which means don't down scale)
807  int original_height = height;
808  if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
809  // Override width and height (but maintain aspect ratio)
810  float ratio = float(width) / float(height);
811  int possible_width = round(max_height * ratio);
812  int possible_height = round(max_width / ratio);
813 
814  if (possible_width <= max_width) {
815  // use calculated width, and max_height
816  width = possible_width;
817  height = max_height;
818  } else {
819  // use max_width, and calculated height
820  width = max_width;
821  height = possible_height;
822  }
823  }
824 
825  // Determine required buffer size and allocate buffer
826  numBytes = avpicture_get_size(PIX_FMT_RGBA, width, height);
827  #pragma omp critical (video_buffer)
828  buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t));
829 
830  // Assign appropriate parts of buffer to image planes in pFrameRGB
831  // Note that pFrameRGB is an AVFrame, but AVFrame is a superset
832  // of AVPicture
833  avpicture_fill((AVPicture *) pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
834 
835  SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, pCodecCtx->pix_fmt, width,
836  height, PIX_FMT_RGBA, SWS_BILINEAR, NULL, NULL, NULL);
837 
838  // Resize / Convert to RGB
839  sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
840  original_height, pFrameRGB->data, pFrameRGB->linesize);
841 
842  // Create or get the existing frame object
843  std::shared_ptr<Frame> f = CreateFrame(current_frame);
844 
845  // Add Image data to frame
846  f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
847 
848  // Update working cache
849  working_cache.Add(f);
850 
851  // Keep track of last last_video_frame
852  #pragma omp critical (video_buffer)
853  last_video_frame = f;
854 
855  // Free the RGB image
856  av_free(buffer);
857  AV_FREE_FRAME(&pFrameRGB);
858 
859  // Remove frame and packet
860  RemoveAVFrame(my_frame);
861  sws_freeContext(img_convert_ctx);
862 
863  // Remove video frame from list of processing video frames
864  {
865  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
866  processing_video_frames.erase(current_frame);
867  processed_video_frames[current_frame] = current_frame;
868  }
869 
870  // Debug output
871  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number, "", -1, "", -1, "", -1);
872 
873  } // end omp task
874 
875 }
876 
877 // Process an audio packet
878 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample)
879 {
880  // Track 1st audio packet after a successful seek
881  if (!seek_audio_frame_found && is_seeking)
882  seek_audio_frame_found = target_frame;
883 
884  // Are we close enough to decode the frame's audio?
885  if (target_frame < (requested_frame - 20))
886  {
887  // Debug output
888  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
889 
890  // Skip to next frame without decoding or caching
891  return;
892  }
893 
894  // Debug output
895  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1);
896 
897  // Init an AVFrame to hold the decoded audio samples
898  int frame_finished = 0;
899  AVFrame *audio_frame = AV_ALLOCATE_FRAME();
900  AV_RESET_FRAME(audio_frame);
901 
902  int packet_samples = 0;
903  int data_size = 0;
904 
905  // re-initialize buffer size (it gets changed in the avcodec_decode_audio2 method call)
906  int buf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE;
907  int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
908 
909  if (frame_finished) {
910 
911  // determine how many samples were decoded
912  int planar = av_sample_fmt_is_planar(aCodecCtx->sample_fmt);
913  int plane_size = -1;
914  data_size = av_samples_get_buffer_size(&plane_size,
915  aCodecCtx->channels,
916  audio_frame->nb_samples,
917  aCodecCtx->sample_fmt, 1);
918 
919  // Calculate total number of samples
920  packet_samples = audio_frame->nb_samples * aCodecCtx->channels;
921  }
922 
923  // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
924  int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
925 
926  // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
927  int64_t adjusted_pts = packet->pts + audio_pts_offset;
928  double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
929  double sample_seconds = double(pts_total) / info.sample_rate;
930 
931  // Debug output
932  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
933  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
934 
935  // DEBUG (FOR AUDIO ISSUES)
936  prev_pts = adjusted_pts;
937  pts_total += pts_remaining_samples;
938  pts_counter++;
939  prev_samples = pts_remaining_samples;
940 
941  // Add audio frame to list of processing audio frames
942  {
943  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
944  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
945  }
946 
947  while (pts_remaining_samples)
948  {
949  // Get Samples per frame (for this frame number)
950  int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
951 
952  // Calculate # of samples to add to this frame
953  int samples = samples_per_frame - previous_packet_location.sample_start;
954  if (samples > pts_remaining_samples)
955  samples = pts_remaining_samples;
956 
957  // Decrement remaining samples
958  pts_remaining_samples -= samples;
959 
960  if (pts_remaining_samples > 0) {
961  // next frame
962  previous_packet_location.frame++;
963  previous_packet_location.sample_start = 0;
964 
965  // Add audio frame to list of processing audio frames
966  {
967  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
968  processing_audio_frames.insert(pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
969  }
970 
971  } else {
972  // Increment sample start
973  previous_packet_location.sample_start += samples;
974  }
975  }
976 
977 
978  // Allocate audio buffer
979  int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE];
980 
981  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", aCodecCtx->sample_fmt, "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16, "", -1);
982 
983  // Create output frame
984  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
985  AV_RESET_FRAME(audio_converted);
986  audio_converted->nb_samples = audio_frame->nb_samples;
987  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
988 
989  AVAudioResampleContext *avr = NULL;
990  int nb_samples = 0;
991 
992  // setup resample context
993  avr = avresample_alloc_context();
994  av_opt_set_int(avr, "in_channel_layout", aCodecCtx->channel_layout, 0);
995  av_opt_set_int(avr, "out_channel_layout", aCodecCtx->channel_layout, 0);
996  av_opt_set_int(avr, "in_sample_fmt", aCodecCtx->sample_fmt, 0);
997  av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
998  av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
999  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1000  av_opt_set_int(avr, "in_channels", info.channels, 0);
1001  av_opt_set_int(avr, "out_channels", info.channels, 0);
1002  int r = avresample_open(avr);
1003 
1004  // Convert audio samples
1005  nb_samples = avresample_convert(avr, // audio resample context
1006  audio_converted->data, // output data pointers
1007  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1008  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1009  audio_frame->data, // input data pointers
1010  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1011  audio_frame->nb_samples); // number of input samples to convert
1012 
1013  // Copy audio samples over original samples
1014  memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1015 
1016  // Deallocate resample buffer
1017  avresample_close(avr);
1018  avresample_free(&avr);
1019  avr = NULL;
1020 
1021  // Free AVFrames
1022  av_free(audio_converted->data[0]);
1023  AV_FREE_FRAME(&audio_converted);
1024 
1025  int64_t starting_frame_number = -1;
1026  bool partial_frame = true;
1027  for (int channel_filter = 0; channel_filter < info.channels; channel_filter++)
1028  {
1029  // Array of floats (to hold samples for each channel)
1030  starting_frame_number = target_frame;
1031  int channel_buffer_size = packet_samples / info.channels;
1032  float *channel_buffer = new float[channel_buffer_size];
1033 
1034  // Init buffer array
1035  for (int z = 0; z < channel_buffer_size; z++)
1036  channel_buffer[z] = 0.0f;
1037 
1038  // Loop through all samples and add them to our Frame based on channel.
1039  // Toggle through each channel number, since channel data is stored like (left right left right)
1040  int channel = 0;
1041  int position = 0;
1042  for (int sample = 0; sample < packet_samples; sample++)
1043  {
1044  // Only add samples for current channel
1045  if (channel_filter == channel)
1046  {
1047  // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1048  channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1049 
1050  // Increment audio position
1051  position++;
1052  }
1053 
1054  // increment channel (if needed)
1055  if ((channel + 1) < info.channels)
1056  // move to next channel
1057  channel ++;
1058  else
1059  // reset channel
1060  channel = 0;
1061  }
1062 
1063  // Loop through samples, and add them to the correct frames
1064  int start = starting_sample;
1065  int remaining_samples = channel_buffer_size;
1066  float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1067  while (remaining_samples > 0)
1068  {
1069  // Get Samples per frame (for this frame number)
1070  int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1071 
1072  // Calculate # of samples to add to this frame
1073  int samples = samples_per_frame - start;
1074  if (samples > remaining_samples)
1075  samples = remaining_samples;
1076 
1077  // Create or get the existing frame object
1078  std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1079 
1080  // Determine if this frame was "partially" filled in
1081  if (samples_per_frame == start + samples)
1082  partial_frame = false;
1083  else
1084  partial_frame = true;
1085 
1086  // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent
1087  // some louder samples from maxing out at 1.0 (not sure why this happens)
1088  f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1089 
1090  // Debug output
1091  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1092 
1093  // Add or update cache
1094  working_cache.Add(f);
1095 
1096  // Decrement remaining samples
1097  remaining_samples -= samples;
1098 
1099  // Increment buffer (to next set of samples)
1100  if (remaining_samples > 0)
1101  iterate_channel_buffer += samples;
1102 
1103  // Increment frame number
1104  starting_frame_number++;
1105 
1106  // Reset starting sample #
1107  start = 0;
1108  }
1109 
1110  // clear channel buffer
1111  delete[] channel_buffer;
1112  channel_buffer = NULL;
1113  iterate_channel_buffer = NULL;
1114  }
1115 
1116  // Clean up some arrays
1117  delete[] audio_buf;
1118  audio_buf = NULL;
1119 
1120  // Remove audio frame from list of processing audio frames
1121  {
1122  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1123  // Update all frames as completed
1124  for (int64_t f = target_frame; f < starting_frame_number; f++) {
1125  // Remove the frame # from the processing list. NOTE: If more than one thread is
1126  // processing this frame, the frame # will be in this list multiple times. We are only
1127  // removing a single instance of it here.
1128  processing_audio_frames.erase(processing_audio_frames.find(f));
1129 
1130  // Check and see if this frame is also being processed by another thread
1131  if (processing_audio_frames.count(f) == 0)
1132  // No other thread is processing it. Mark the audio as processed (final)
1133  processed_audio_frames[f] = f;
1134  }
1135 
1136  if (target_frame == starting_frame_number) {
1137  // This typically never happens, but just in case, remove the currently processing number
1138  processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1139  }
1140  }
1141 
1142  // Free audio frame
1143  AV_FREE_FRAME(&audio_frame);
1144 
1145  // Debug output
1146  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1, "", -1, "", -1, "", -1);
1147 
1148 }
1149 
1150 
1151 
1152 // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1153 void FFmpegReader::Seek(int64_t requested_frame)
1154 {
1155  // Adjust for a requested frame that is too small or too large
1156  if (requested_frame < 1)
1157  requested_frame = 1;
1158  if (requested_frame > info.video_length)
1159  requested_frame = info.video_length;
1160 
1161  int processing_video_frames_size = 0;
1162  int processing_audio_frames_size = 0;
1163  {
1164  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1165  processing_video_frames_size = processing_video_frames.size();
1166  processing_audio_frames_size = processing_audio_frames.size();
1167  }
1168 
1169  // Debug output
1170  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "", -1);
1171 
1172  // Wait for any processing frames to complete
1173  while (processing_video_frames_size + processing_audio_frames_size > 0) {
1174  usleep(2500);
1175  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1176  processing_video_frames_size = processing_video_frames.size();
1177  processing_audio_frames_size = processing_audio_frames.size();
1178  }
1179 
1180  // Clear working cache (since we are seeking to another location in the file)
1181  working_cache.Clear();
1182  missing_frames.Clear();
1183 
1184  // Clear processed lists
1185  {
1186  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1187  processing_audio_frames.clear();
1188  processing_video_frames.clear();
1189  processed_video_frames.clear();
1190  processed_audio_frames.clear();
1191  missing_audio_frames.clear();
1192  missing_video_frames.clear();
1193  missing_audio_frames_source.clear();
1194  missing_video_frames_source.clear();
1195  checked_frames.clear();
1196  }
1197 
1198  // Reset the last frame variable
1199  last_frame = 0;
1200  current_video_frame = 0;
1201  largest_frame_processed = 0;
1202  num_checks_since_final = 0;
1203  num_packets_since_video_frame = 0;
1204  has_missing_frames = false;
1205  bool has_audio_override = info.has_audio;
1206  bool has_video_override = info.has_video;
1207 
1208  // Increment seek count
1209  seek_count++;
1210 
1211  // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1212  int buffer_amount = 6;
1213  if (requested_frame - buffer_amount < 20)
1214  {
1215  // Close and re-open file (basically seeking to frame 1)
1216  Close();
1217  Open();
1218 
1219  // Update overrides (since closing and re-opening might update these)
1220  info.has_audio = has_audio_override;
1221  info.has_video = has_video_override;
1222 
1223  // Not actually seeking, so clear these flags
1224  is_seeking = false;
1225  if (seek_count == 1) {
1226  // Don't redefine this on multiple seek attempts for a specific frame
1227  seeking_frame = 1;
1228  seeking_pts = ConvertFrameToVideoPTS(1);
1229  }
1230  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1231  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1232  }
1233  else
1234  {
1235  // Seek to nearest key-frame (aka, i-frame)
1236  bool seek_worked = false;
1237  int64_t seek_target = 0;
1238 
1239  // Seek video stream (if any)
1240  if (!seek_worked && info.has_video)
1241  {
1242  seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1243  if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1244  fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->filename);
1245  } else
1246  {
1247  // VIDEO SEEK
1248  is_video_seek = true;
1249  seek_worked = true;
1250  }
1251  }
1252 
1253  // Seek audio stream (if not already seeked... and if an audio stream is found)
1254  if (!seek_worked && info.has_audio)
1255  {
1256  seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1257  if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1258  fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->filename);
1259  } else
1260  {
1261  // AUDIO SEEK
1262  is_video_seek = false;
1263  seek_worked = true;
1264  }
1265  }
1266 
1267  // Was the seek successful?
1268  if (seek_worked)
1269  {
1270  // Flush audio buffer
1271  if (info.has_audio)
1272  avcodec_flush_buffers(aCodecCtx);
1273 
1274  // Flush video buffer
1275  if (info.has_video)
1276  avcodec_flush_buffers(pCodecCtx);
1277 
1278  // Reset previous audio location to zero
1279  previous_packet_location.frame = -1;
1280  previous_packet_location.sample_start = 0;
1281 
1282  // init seek flags
1283  is_seeking = true;
1284  if (seek_count == 1) {
1285  // Don't redefine this on multiple seek attempts for a specific frame
1286  seeking_pts = seek_target;
1287  seeking_frame = requested_frame;
1288  }
1289  seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1290  seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1291 
1292  }
1293  else
1294  {
1295  // seek failed
1296  is_seeking = false;
1297  seeking_pts = 0;
1298  seeking_frame = 0;
1299 
1300  // dislable seeking for this reader (since it failed)
1301  // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1302  enable_seek = false;
1303 
1304  // Close and re-open file (basically seeking to frame 1)
1305  Close();
1306  Open();
1307 
1308  // Update overrides (since closing and re-opening might update these)
1309  info.has_audio = has_audio_override;
1310  info.has_video = has_video_override;
1311  }
1312  }
1313 }
1314 
1315 // Get the PTS for the current video packet
1316 int64_t FFmpegReader::GetVideoPTS()
1317 {
1318  int64_t current_pts = 0;
1319  if(packet->dts != AV_NOPTS_VALUE)
1320  current_pts = packet->dts;
1321 
1322  // Return adjusted PTS
1323  return current_pts;
1324 }
1325 
1326 // Update PTS Offset (if any)
1327 void FFmpegReader::UpdatePTSOffset(bool is_video)
1328 {
1329  // Determine the offset between the PTS and Frame number (only for 1st frame)
1330  if (is_video)
1331  {
1332  // VIDEO PACKET
1333  if (video_pts_offset == 99999) // Has the offset been set yet?
1334  {
1335  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1336  video_pts_offset = 0 - max(GetVideoPTS(), (int64_t) info.video_timebase.ToInt() * 10);
1337 
1338  // debug output
1339  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1);
1340  }
1341  }
1342  else
1343  {
1344  // AUDIO PACKET
1345  if (audio_pts_offset == 99999) // Has the offset been set yet?
1346  {
1347  // Find the difference between PTS and frame number (no more than 10 timebase units allowed)
1348  audio_pts_offset = 0 - max(packet->pts, (int64_t) info.audio_timebase.ToInt() * 10);
1349 
1350  // debug output
1351  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Audio)", "audio_pts_offset", audio_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1);
1352  }
1353  }
1354 }
1355 
1356 // Convert PTS into Frame Number
1357 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts)
1358 {
1359  // Apply PTS offset
1360  pts = pts + video_pts_offset;
1361  int64_t previous_video_frame = current_video_frame;
1362 
1363  // Get the video packet start time (in seconds)
1364  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1365 
1366  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1367  int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
1368 
1369  // Keep track of the expected video frame #
1370  if (current_video_frame == 0)
1371  current_video_frame = frame;
1372  else {
1373 
1374  // Sometimes frames are duplicated due to identical (or similar) timestamps
1375  if (frame == previous_video_frame) {
1376  // return -1 frame number
1377  frame = -1;
1378  }
1379  else
1380  // Increment expected frame
1381  current_video_frame++;
1382 
1383  if (current_video_frame < frame)
1384  // has missing frames
1385  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame, "", -1, "", -1, "", -1);
1386 
1387  // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1388  // if we are missing a video frame.
1389  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1390  while (current_video_frame < frame) {
1391  if (!missing_video_frames.count(current_video_frame)) {
1392  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame, "", -1, "", -1, "", -1, "", -1);
1393  missing_video_frames.insert(pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1394  missing_video_frames_source.insert(pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1395  }
1396 
1397  // Mark this reader as containing missing frames
1398  has_missing_frames = true;
1399 
1400  // Increment current frame
1401  current_video_frame++;
1402  }
1403  }
1404 
1405  // Return frame #
1406  return frame;
1407 }
1408 
1409 // Convert Frame Number into Video PTS
1410 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number)
1411 {
1412  // Get timestamp of this frame (in seconds)
1413  double seconds = double(frame_number) / info.fps.ToDouble();
1414 
1415  // Calculate the # of video packets in this timestamp
1416  int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
1417 
1418  // Apply PTS offset (opposite)
1419  return video_pts - video_pts_offset;
1420 }
1421 
1422 // Convert Frame Number into Video PTS
1423 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number)
1424 {
1425  // Get timestamp of this frame (in seconds)
1426  double seconds = double(frame_number) / info.fps.ToDouble();
1427 
1428  // Calculate the # of audio packets in this timestamp
1429  int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
1430 
1431  // Apply PTS offset (opposite)
1432  return audio_pts - audio_pts_offset;
1433 }
1434 
1435 // Calculate Starting video frame and sample # for an audio PTS
1436 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
1437 {
1438  // Apply PTS offset
1439  pts = pts + audio_pts_offset;
1440 
1441  // Get the audio packet start time (in seconds)
1442  double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1443 
1444  // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1445  double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1446 
1447  // Frame # as a whole number (no more decimals)
1448  int64_t whole_frame = int64_t(frame);
1449 
1450  // Remove the whole number, and only get the decimal of the frame
1451  double sample_start_percentage = frame - double(whole_frame);
1452 
1453  // Get Samples per frame
1454  int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1455 
1456  // Calculate the sample # to start on
1457  int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1458 
1459  // Protect against broken (i.e. negative) timestamps
1460  if (whole_frame < 1)
1461  whole_frame = 1;
1462  if (sample_start < 0)
1463  sample_start = 0;
1464 
1465  // Prepare final audio packet location
1466  AudioLocation location = {whole_frame, sample_start};
1467 
1468  // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1469  if (previous_packet_location.frame != -1) {
1470  if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1471  {
1472  int64_t orig_frame = location.frame;
1473  int orig_start = location.sample_start;
1474 
1475  // Update sample start, to prevent gaps in audio
1476  location.sample_start = previous_packet_location.sample_start;
1477  location.frame = previous_packet_location.frame;
1478 
1479  // Debug output
1480  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1);
1481 
1482  } else {
1483  // Debug output
1484  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1, "", -1);
1485 
1486  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1487  for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
1488  if (!missing_audio_frames.count(audio_frame)) {
1489  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame, "", -1, "", -1, "", -1);
1490  missing_audio_frames.insert(pair<int64_t, int64_t>(previous_packet_location.frame - 1, audio_frame));
1491  }
1492  }
1493  }
1494  }
1495 
1496  // Set previous location
1497  previous_packet_location = location;
1498 
1499  // Return the associated video frame and starting sample #
1500  return location;
1501 }
1502 
1503 // Create a new Frame (or return an existing one) and add it to the working queue.
1504 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame)
1505 {
1506  // Check working cache
1507  std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
1508  if (!output)
1509  {
1510  // Create a new frame on the working cache
1511  output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
1512  output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
1513  output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
1514  output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
1515 
1516  working_cache.Add(output);
1517 
1518  // Set the largest processed frame (if this is larger)
1519  if (requested_frame > largest_frame_processed)
1520  largest_frame_processed = requested_frame;
1521  }
1522 
1523  // Return new frame
1524  return output;
1525 }
1526 
1527 // Determine if frame is partial due to seek
1528 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
1529 
1530  // Sometimes a seek gets partial frames, and we need to remove them
1531  bool seek_trash = false;
1532  int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1533  if (seek_video_frame_found > max_seeked_frame)
1534  max_seeked_frame = seek_video_frame_found;
1535  if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1536  (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1537  seek_trash = true;
1538 
1539  return seek_trash;
1540 }
1541 
1542 // Check if a frame is missing and attempt to replace it's frame image (and
1543 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
1544 {
1545  // Lock
1546  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1547 
1548  // Init # of times this frame has been checked so far
1549  int checked_count = 0;
1550 
1551  // Increment check count for this frame (or init to 1)
1552  if (checked_frames.count(requested_frame) == 0)
1553  checked_frames[requested_frame] = 1;
1554  else
1555  checked_frames[requested_frame]++;
1556  checked_count = checked_frames[requested_frame];
1557 
1558  // Debug output
1559  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_count, "", -1, "", -1);
1560 
1561 
1562  // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
1563  map<int64_t, int64_t>::iterator itr;
1564  bool found_missing_frame = false;
1565 
1566  // Check if requested frame is a missing frame
1567  if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
1568  int64_t missing_source_frame = -1;
1569  if (missing_video_frames.count(requested_frame))
1570  missing_source_frame = missing_video_frames.find(requested_frame)->second;
1571  else if (missing_audio_frames.count(requested_frame))
1572  missing_source_frame = missing_audio_frames.find(requested_frame)->second;
1573 
1574  // Increment missing source frame check count (or init to 1)
1575  if (checked_frames.count(missing_source_frame) == 0)
1576  checked_frames[missing_source_frame] = 1;
1577  else
1578  checked_frames[missing_source_frame]++;
1579 
1580  // Get the previous frame of this missing frame (if it's available in missing cache)
1581  std::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
1582  if (parent_frame == NULL) {
1583  parent_frame = final_cache.GetFrame(missing_source_frame);
1584  if (parent_frame != NULL) {
1585  // Add missing final frame to missing cache
1586  missing_frames.Add(parent_frame);
1587  }
1588  }
1589 
1590  // Create blank missing frame
1591  std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1592 
1593  // Debug output
1594  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1595 
1596  // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
1597  if (parent_frame != NULL) {
1598  // Debug output
1599  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame, "", -1, "", -1, "", -1);
1600 
1601  // Add this frame to the processed map (since it's already done)
1602  std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
1603  if (parent_image) {
1604  missing_frame->AddImage(std::shared_ptr<QImage>(new QImage(*parent_image)));
1605 
1606  processed_video_frames[missing_frame->number] = missing_frame->number;
1607  processed_audio_frames[missing_frame->number] = missing_frame->number;
1608 
1609  // Move frame to final cache
1610  final_cache.Add(missing_frame);
1611 
1612  // Remove frame from working cache
1613  working_cache.Remove(missing_frame->number);
1614 
1615  // Update last_frame processed
1616  last_frame = missing_frame->number;
1617  }
1618  }
1619 
1620  }
1621 
1622  return found_missing_frame;
1623 }
1624 
1625 // Check the working queue, and move finished frames to the finished queue
1626 void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame)
1627 {
1628  // Loop through all working queue frames
1629  bool checked_count_tripped = false;
1630  int max_checked_count = 80;
1631 
1632  while (true)
1633  {
1634  // Get the front frame of working cache
1635  std::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
1636 
1637  // Was a frame found?
1638  if (!f)
1639  // No frames found
1640  break;
1641 
1642  // Check if this frame is 'missing'
1643  CheckMissingFrame(f->number);
1644 
1645  // Init # of times this frame has been checked so far
1646  int checked_count = 0;
1647  int checked_frames_size = 0;
1648 
1649  bool is_video_ready = false;
1650  bool is_audio_ready = false;
1651  { // limit scope of next few lines
1652  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1653  is_video_ready = processed_video_frames.count(f->number);
1654  is_audio_ready = processed_audio_frames.count(f->number);
1655 
1656  // Get check count for this frame
1657  checked_frames_size = checked_frames.size();
1658  if (!checked_count_tripped || f->number >= requested_frame)
1659  checked_count = checked_frames[f->number];
1660  else
1661  // Force checked count over the limit
1662  checked_count = max_checked_count;
1663  }
1664 
1665  if (previous_packet_location.frame == f->number && !end_of_stream)
1666  is_audio_ready = false; // don't finalize the last processed audio frame
1667  bool is_seek_trash = IsPartialFrame(f->number);
1668 
1669  // Adjust for available streams
1670  if (!info.has_video) is_video_ready = true;
1671  if (!info.has_audio) is_audio_ready = true;
1672 
1673  // Make final any frames that get stuck (for whatever reason)
1674  if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
1675  // Debug output
1676  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
1677 
1678  // Trigger checked count tripped mode (clear out all frames before requested frame)
1679  checked_count_tripped = true;
1680 
1681  if (info.has_video && !is_video_ready && last_video_frame) {
1682  // Copy image from last frame
1683  f->AddImage(std::shared_ptr<QImage>(new QImage(*last_video_frame->GetImage())));
1684  is_video_ready = true;
1685  }
1686 
1687  if (info.has_audio && !is_audio_ready) {
1688  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1689  // Mark audio as processed, and indicate the frame has audio data
1690  is_audio_ready = true;
1691  }
1692  }
1693 
1694  // Debug output
1695  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
1696 
1697  // Check if working frame is final
1698  if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash)
1699  {
1700  // Debug output
1701  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1);
1702 
1703  if (!is_seek_trash)
1704  {
1705  // Reset counter since last 'final' frame
1706  num_checks_since_final = 0;
1707 
1708  // Move frame to final cache
1709  final_cache.Add(f);
1710 
1711  // Add to missing cache (if another frame depends on it)
1712  {
1713  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1714  if (missing_video_frames_source.count(f->number)) {
1715  // Debug output
1716  ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1);
1717  missing_frames.Add(f);
1718  }
1719 
1720  // Remove from 'checked' count
1721  checked_frames.erase(f->number);
1722  }
1723 
1724  // Remove frame from working cache
1725  working_cache.Remove(f->number);
1726 
1727  // Update last frame processed
1728  last_frame = f->number;
1729 
1730  } else {
1731  // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
1732  working_cache.Remove(f->number);
1733  }
1734  }
1735  else
1736  // Stop looping
1737  break;
1738  }
1739 }
1740 
1741 // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
1742 void FFmpegReader::CheckFPS()
1743 {
1744  check_fps = true;
1745  avpicture_alloc(pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1746 
1747  int first_second_counter = 0;
1748  int second_second_counter = 0;
1749  int third_second_counter = 0;
1750  int forth_second_counter = 0;
1751  int fifth_second_counter = 0;
1752 
1753  int iterations = 0;
1754  int threshold = 500;
1755 
1756  // Loop through the stream
1757  while (true)
1758  {
1759  // Get the next packet (if any)
1760  if (GetNextPacket() < 0)
1761  // Break loop when no more packets found
1762  break;
1763 
1764  // Video packet
1765  if (packet->stream_index == videoStream)
1766  {
1767  // Check if the AVFrame is finished and set it
1768  if (GetAVFrame())
1769  {
1770  // Update PTS / Frame Offset (if any)
1771  UpdatePTSOffset(true);
1772 
1773  // Get PTS of this packet
1774  int64_t pts = GetVideoPTS();
1775 
1776  // Remove pFrame
1777  RemoveAVFrame(pFrame);
1778 
1779  // Apply PTS offset
1780  pts += video_pts_offset;
1781 
1782  // Get the video packet start time (in seconds)
1783  double video_seconds = double(pts) * info.video_timebase.ToDouble();
1784 
1785  // Increment the correct counter
1786  if (video_seconds <= 1.0)
1787  first_second_counter++;
1788  else if (video_seconds > 1.0 && video_seconds <= 2.0)
1789  second_second_counter++;
1790  else if (video_seconds > 2.0 && video_seconds <= 3.0)
1791  third_second_counter++;
1792  else if (video_seconds > 3.0 && video_seconds <= 4.0)
1793  forth_second_counter++;
1794  else if (video_seconds > 4.0 && video_seconds <= 5.0)
1795  fifth_second_counter++;
1796  else
1797  // Too far
1798  break;
1799  }
1800  }
1801 
1802  // Increment counters
1803  iterations++;
1804 
1805  // Give up (if threshold exceeded)
1806  if (iterations > threshold)
1807  break;
1808  }
1809 
1810  // Double check that all counters have greater than zero (or give up)
1811  if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0)
1812  {
1813  // Seek to frame 1
1814  Seek(1);
1815 
1816  // exit with no changes to FPS (not enough data to calculate)
1817  return;
1818  }
1819 
1820  int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1821  int avg_fps = round(sum_fps / 4.0f);
1822 
1823  // Sometimes the FPS is incorrectly detected by FFmpeg. If the 1st and 2nd seconds counters
1824  // agree with each other, we are going to adjust the FPS of this reader instance. Otherwise, print
1825  // a warning message.
1826 
1827  // Get diff from actual frame rate
1828  double fps = info.fps.ToDouble();
1829  double diff = fps - double(avg_fps);
1830 
1831  // Is difference bigger than 1 frame?
1832  if (diff <= -1 || diff >= 1)
1833  {
1834  // Compare to half the frame rate (the most common type of issue)
1835  double half_fps = Fraction(info.fps.num / 2, info.fps.den).ToDouble();
1836  diff = half_fps - double(avg_fps);
1837 
1838  // Is difference bigger than 1 frame?
1839  if (diff <= -1 || diff >= 1)
1840  {
1841  // Update FPS for this reader instance
1842  info.fps = Fraction(avg_fps, 1);
1843  }
1844  else
1845  {
1846  // Update FPS for this reader instance (to 1/2 the original framerate)
1847  info.fps = Fraction(info.fps.num / 2, info.fps.den);
1848  }
1849  }
1850 
1851  // Seek to frame 1
1852  Seek(1);
1853 }
1854 
1855 // Remove AVFrame from cache (and deallocate it's memory)
1856 void FFmpegReader::RemoveAVFrame(AVPicture* remove_frame)
1857 {
1858  // Remove pFrame (if exists)
1859  if (remove_frame)
1860  {
1861  // Free memory
1862  avpicture_free(remove_frame);
1863 
1864  // Delete the object
1865  delete remove_frame;
1866  }
1867 }
1868 
1869 // Remove AVPacket from cache (and deallocate it's memory)
1870 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1871 {
1872  // deallocate memory for packet
1873  AV_FREE_PACKET(remove_packet);
1874 
1875  // Delete the object
1876  delete remove_packet;
1877 }
1878 
1879 /// Get the smallest video frame that is still being processed
1880 int64_t FFmpegReader::GetSmallestVideoFrame()
1881 {
1882  // Loop through frame numbers
1883  map<int64_t, int64_t>::iterator itr;
1884  int64_t smallest_frame = -1;
1885  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1886  for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1887  {
1888  if (itr->first < smallest_frame || smallest_frame == -1)
1889  smallest_frame = itr->first;
1890  }
1891 
1892  // Return frame number
1893  return smallest_frame;
1894 }
1895 
1896 /// Get the smallest audio frame that is still being processed
1897 int64_t FFmpegReader::GetSmallestAudioFrame()
1898 {
1899  // Loop through frame numbers
1900  map<int64_t, int64_t>::iterator itr;
1901  int64_t smallest_frame = -1;
1902  const GenericScopedLock<CriticalSection> lock(processingCriticalSection);
1903  for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
1904  {
1905  if (itr->first < smallest_frame || smallest_frame == -1)
1906  smallest_frame = itr->first;
1907  }
1908 
1909  // Return frame number
1910  return smallest_frame;
1911 }
1912 
1913 // Generate JSON string of this object
1915 
1916  // Return formatted string
1917  return JsonValue().toStyledString();
1918 }
1919 
1920 // Generate Json::JsonValue for this object
1922 
1923  // Create root json object
1924  Json::Value root = ReaderBase::JsonValue(); // get parent properties
1925  root["type"] = "FFmpegReader";
1926  root["path"] = path;
1927 
1928  // return JsonValue
1929  return root;
1930 }
1931 
1932 // Load JSON string into this object
1933 void FFmpegReader::SetJson(string value) {
1934 
1935  // Parse JSON string into JSON objects
1936  Json::Value root;
1937  Json::Reader reader;
1938  bool success = reader.parse( value, root );
1939  if (!success)
1940  // Raise exception
1941  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
1942 
1943  try
1944  {
1945  // Set all values that match
1946  SetJsonValue(root);
1947  }
1948  catch (exception e)
1949  {
1950  // Error parsing JSON (or missing keys)
1951  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
1952  }
1953 }
1954 
1955 // Load Json::JsonValue into this object
1956 void FFmpegReader::SetJsonValue(Json::Value root) {
1957 
1958  // Set parent data
1960 
1961  // Set data from Json (if key is found)
1962  if (!root["path"].isNull())
1963  path = root["path"].asString();
1964 
1965  // Re-Open path, and re-init everything (if needed)
1966  if (is_open)
1967  {
1968  Close();
1969  Open();
1970  }
1971 }
#define AV_RESET_FRAME(av_frame)
int max_height
The maximium image height needed by this clip (used for optimizations)
Definition: ReaderBase.h:103
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
Definition: Fraction.h:44
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
Definition: ReaderBase.h:99
CriticalSection processingCriticalSection
Definition: ReaderBase.h:100
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:83
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:67
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:41
float duration
Length of time (in seconds)
Definition: ReaderBase.h:64
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:79
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:46
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:234
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:61
void Close()
Close File.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: ReaderBase.h:72
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:65
FFmpegReader(string path)
std::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:80
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:62
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:84
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:74
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
Definition: Exceptions.h:192
int height
The height of the video (in pixels)
Definition: ReaderBase.h:66
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: ReaderBase.cpp:106
#define PixelFormat
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: ReaderBase.cpp:155
ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Definition: Fraction.cpp:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:69
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:76
Exception for frames that are out of bounds.
Definition: Exceptions.h:202
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
int64_t Count()
Count the frames in the queue.
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: ReaderBase.h:71
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
Definition: Exceptions.h:152
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:68
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:59
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:70
#define PIX_FMT_RGBA
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:85
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:73
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:222
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
Definition: Fraction.h:45
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:82
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:75
int max_width
The maximum image width needed by this clip (used for optimizations)
Definition: ReaderBase.h:102
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:505
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:81