31 #include "../include/FFmpegReader.h" 36 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
37 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
38 check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
39 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
40 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), packet(NULL) {
44 avcodec_register_all();
57 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
58 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
59 check_fps(false),
enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
60 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
61 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), packet(NULL) {
65 avcodec_register_all();
89 if (abs(location.
frame - frame) >= 2)
95 int64_t diff = samples_per_frame * (location.
frame - frame) + location.
sample_start - sample_start;
96 if (abs(diff) <= amount)
113 if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
114 throw InvalidFile(
"File could not be opened.", path);
117 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
123 for (
unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
126 if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
130 if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
134 if (videoStream == -1 && audioStream == -1)
135 throw NoStreamsFound(
"No video or audio streams found in this file.", path);
138 if (videoStream != -1)
144 pStream = pFormatCtx->streams[videoStream];
145 pCodecCtx = pFormatCtx->streams[videoStream]->codec;
151 AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
152 if (pCodec == NULL) {
153 throw InvalidCodec(
"A valid video codec could not be found for this file.", path);
156 if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
157 throw InvalidCodec(
"A video codec was found, but could not be opened.", path);
164 if (audioStream != -1)
170 aStream = pFormatCtx->streams[audioStream];
171 aCodecCtx = pFormatCtx->streams[audioStream]->codec;
177 AVCodec *aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
178 if (aCodec == NULL) {
179 throw InvalidCodec(
"A valid audio codec could not be found for this file.", path);
182 if (avcodec_open2(aCodecCtx, aCodec, NULL) < 0)
183 throw InvalidCodec(
"An audio codec was found, but could not be opened.", path);
190 previous_packet_location.
frame = -1;
211 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Close",
"", -1,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
216 avcodec_flush_buffers(pCodecCtx);
217 avcodec_close(pCodecCtx);
221 avcodec_flush_buffers(aCodecCtx);
222 avcodec_close(aCodecCtx);
227 working_cache.
Clear();
228 missing_frames.
Clear();
233 processed_video_frames.clear();
234 processed_audio_frames.clear();
235 processing_video_frames.clear();
236 processing_audio_frames.clear();
237 missing_audio_frames.clear();
238 missing_video_frames.clear();
239 missing_audio_frames_source.clear();
240 missing_video_frames_source.clear();
241 checked_frames.clear();
245 avformat_close_input(&pFormatCtx);
246 av_freep(&pFormatCtx);
250 largest_frame_processed = 0;
251 seek_audio_frame_found = 0;
252 seek_video_frame_found = 0;
253 current_video_frame = 0;
254 has_missing_frames =
false;
258 void FFmpegReader::UpdateAudioInfo()
262 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
265 if (aCodecCtx->channel_layout == 0)
266 aCodecCtx->channel_layout = av_get_default_channel_layout( aCodecCtx->channels );
276 if (aStream->duration > 0.0f && aStream->duration >
info.
duration)
302 void FFmpegReader::UpdateVideoInfo()
306 info.
file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
318 if (pStream->sample_aspect_ratio.num != 0)
323 else if (pCodecCtx->sample_aspect_ratio.num != 0)
369 is_duration_known =
false;
374 is_duration_known =
true;
400 throw ReaderClosed(
"The FFmpegReader is closed. Call Open() before calling this method.", path);
403 if (requested_frame < 1)
409 throw InvalidFile(
"Could not detect the duration of the video or audio stream.", path);
412 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"requested_frame", requested_frame,
"last_frame", last_frame,
"", -1,
"", -1,
"", -1,
"", -1);
418 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
429 if (has_missing_frames)
430 CheckMissingFrame(requested_frame);
434 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetFrame",
"returned cached frame on 2nd look", requested_frame,
"", -1,
"", -1,
"", -1,
"", -1,
"", -1);
445 if (last_frame == 0 && requested_frame != 1)
450 int64_t diff = requested_frame - last_frame;
451 if (diff >= 1 && diff <= 20)
454 return ReadStream(requested_frame);
461 Seek(requested_frame);
471 return ReadStream(requested_frame);
478 std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame)
481 bool end_of_stream =
false;
482 bool check_seek =
false;
483 bool frame_finished =
false;
484 int packet_error = -1;
487 int packets_processed = 0;
489 int max_packets = 4096;
494 omp_set_nested(
true);
497 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream",
"requested_frame", requested_frame,
"OPEN_MP_NUM_PROCESSORS",
OPEN_MP_NUM_PROCESSORS,
"", -1,
"", -1,
"", -1,
"", -1);
507 packet_error = GetNextPacket();
509 int processing_video_frames_size = 0;
510 int processing_audio_frames_size = 0;
513 processing_video_frames_size = processing_video_frames.size();
514 processing_audio_frames_size = processing_audio_frames.size();
518 while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
521 processing_video_frames_size = processing_video_frames.size();
522 processing_audio_frames_size = processing_audio_frames.size();
526 if (packet_error < 0)
529 end_of_stream =
true;
534 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (GetNextPacket)",
"requested_frame", requested_frame,
"processing_video_frames_size", processing_video_frames_size,
"processing_audio_frames_size", processing_audio_frames_size,
"minimum_packets", minimum_packets,
"packets_processed", packets_processed,
"is_seeking", is_seeking);
540 num_packets_since_video_frame = 0;
544 #pragma omp critical (openshot_seek) 545 check_seek = CheckSeek(
true);
555 frame_finished = GetAVFrame();
561 UpdatePTSOffset(
true);
564 ProcessVideoPacket(requested_frame);
569 else if (
info.
has_audio && packet->stream_index == audioStream)
572 num_packets_since_video_frame++;
576 #pragma omp critical (openshot_seek) 577 check_seek = CheckSeek(
false);
587 UpdatePTSOffset(
false);
597 bool is_cache_found =
false;
600 CheckMissingFrame(requested_frame);
603 CheckWorkingFrames(
false, requested_frame);
613 if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
622 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ReadStream (Completed)",
"packets_processed", packets_processed,
"end_of_stream", end_of_stream,
"largest_frame_processed", largest_frame_processed,
"Working Cache Count", working_cache.
Count(),
"", -1,
"", -1);
627 CheckWorkingFrames(end_of_stream, requested_frame);
644 std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
653 int FFmpegReader::GetNextPacket()
655 int found_packet = 0;
656 AVPacket *next_packet =
new AVPacket();
657 found_packet = av_read_frame(pFormatCtx, next_packet);
661 RemoveAVPacket(packet);
665 if (found_packet >= 0)
668 packet = next_packet;
676 bool FFmpegReader::GetAVFrame()
678 int frameFinished = -1;
682 #pragma omp critical (packet_cache) 683 avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
690 pFrame =
new AVPicture();
692 av_picture_copy(pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt,
info.
width,
info.
height);
695 if (!check_interlace)
697 check_interlace =
true;
707 return frameFinished;
711 bool FFmpegReader::CheckSeek(
bool is_video)
718 if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
726 int64_t max_seeked_frame = seek_audio_frame_found;
727 if (seek_video_frame_found > max_seeked_frame)
728 max_seeked_frame = seek_video_frame_found;
731 if (max_seeked_frame >= seeking_frame)
734 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Too far, seek again)",
"is_video_seek", is_video_seek,
"max_seeked_frame", max_seeked_frame,
"seeking_frame", seeking_frame,
"seeking_pts", seeking_pts,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
737 Seek(seeking_frame - (20 * seek_count * seek_count));
742 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckSeek (Successful)",
"is_video_seek", is_video_seek,
"current_pts", packet->pts,
"seeking_pts", seeking_pts,
"seeking_frame", seeking_frame,
"seek_video_frame_found", seek_video_frame_found,
"seek_audio_frame_found", seek_audio_frame_found);
756 void FFmpegReader::ProcessVideoPacket(int64_t requested_frame)
759 int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
762 if (!seek_video_frame_found && is_seeking)
763 seek_video_frame_found = current_frame;
766 if ((current_frame < (requested_frame - 20)) or (current_frame == -1))
769 RemoveAVFrame(pFrame);
772 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Skipped)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
779 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (Before)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"", -1,
"", -1,
"", -1,
"", -1);
786 AVPicture *my_frame = pFrame;
790 processing_video_frames[current_frame] = current_frame;
792 #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt) 795 AVFrame *pFrameRGB = NULL;
797 uint8_t *buffer = NULL;
801 if (pFrameRGB == NULL)
802 throw OutOfBoundsFrame(
"Convert Image Broke!", current_frame, video_length);
807 int original_height = height;
810 float ratio = float(width) / float(height);
811 int possible_width = round(
max_height * ratio);
812 int possible_height = round(
max_width / ratio);
816 width = possible_width;
821 height = possible_height;
826 numBytes = avpicture_get_size(
PIX_FMT_RGBA, width, height);
827 #pragma omp critical (video_buffer) 828 buffer = (uint8_t *) av_malloc(numBytes *
sizeof(uint8_t));
833 avpicture_fill((AVPicture *) pFrameRGB, buffer,
PIX_FMT_RGBA, width, height);
835 SwsContext *img_convert_ctx = sws_getContext(
info.
width,
info.
height, pCodecCtx->pix_fmt, width,
839 sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
840 original_height, pFrameRGB->data, pFrameRGB->linesize);
843 std::shared_ptr<Frame> f = CreateFrame(current_frame);
846 f->AddImage(width, height, 4, QImage::Format_RGBA8888, buffer);
849 working_cache.
Add(f);
852 #pragma omp critical (video_buffer) 853 last_video_frame = f;
860 RemoveAVFrame(my_frame);
861 sws_freeContext(img_convert_ctx);
866 processing_video_frames.erase(current_frame);
867 processed_video_frames[current_frame] = current_frame;
871 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessVideoPacket (After)",
"requested_frame", requested_frame,
"current_frame", current_frame,
"f->number", f->number,
"", -1,
"", -1,
"", -1);
878 void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame,
int starting_sample)
881 if (!seek_audio_frame_found && is_seeking)
882 seek_audio_frame_found = target_frame;
885 if (target_frame < (requested_frame - 20))
888 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Skipped)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
895 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Before)",
"requested_frame", requested_frame,
"target_frame", target_frame,
"starting_sample", starting_sample,
"", -1,
"", -1,
"", -1);
898 int frame_finished = 0;
902 int packet_samples = 0;
907 int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
909 if (frame_finished) {
912 int planar = av_sample_fmt_is_planar(aCodecCtx->sample_fmt);
914 data_size = av_samples_get_buffer_size(&plane_size,
916 audio_frame->nb_samples,
917 aCodecCtx->sample_fmt, 1);
920 packet_samples = audio_frame->nb_samples * aCodecCtx->channels;
924 int pts_remaining_samples = packet_samples /
info.
channels;
927 int64_t adjusted_pts = packet->pts + audio_pts_offset;
932 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info A)",
"pts_counter", pts_counter,
"PTS", adjusted_pts,
"Offset", audio_pts_offset,
"PTS Diff", adjusted_pts - prev_pts,
"Samples", pts_remaining_samples,
"Sample PTS ratio",
float(adjusted_pts - prev_pts) / pts_remaining_samples);
933 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (Decode Info B)",
"Sample Diff", pts_remaining_samples - prev_samples - prev_pts,
"Total", pts_total,
"PTS Seconds", audio_seconds,
"Sample Seconds", sample_seconds,
"Seconds Diff", audio_seconds - sample_seconds,
"raw samples", packet_samples);
936 prev_pts = adjusted_pts;
937 pts_total += pts_remaining_samples;
939 prev_samples = pts_remaining_samples;
944 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
947 while (pts_remaining_samples)
953 int samples = samples_per_frame - previous_packet_location.
sample_start;
954 if (samples > pts_remaining_samples)
955 samples = pts_remaining_samples;
958 pts_remaining_samples -= samples;
960 if (pts_remaining_samples > 0) {
962 previous_packet_location.
frame++;
968 processing_audio_frames.insert(pair<int, int>(previous_packet_location.
frame, previous_packet_location.
frame));
981 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (ReSample)",
"packet_samples", packet_samples,
"info.channels",
info.
channels,
"info.sample_rate",
info.
sample_rate,
"aCodecCtx->sample_fmt", aCodecCtx->sample_fmt,
"AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16,
"", -1);
986 audio_converted->nb_samples = audio_frame->nb_samples;
987 av_samples_alloc(audio_converted->data, audio_converted->linesize,
info.
channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
989 AVAudioResampleContext *avr = NULL;
993 avr = avresample_alloc_context();
994 av_opt_set_int(avr,
"in_channel_layout", aCodecCtx->channel_layout, 0);
995 av_opt_set_int(avr,
"out_channel_layout", aCodecCtx->channel_layout, 0);
996 av_opt_set_int(avr,
"in_sample_fmt", aCodecCtx->sample_fmt, 0);
997 av_opt_set_int(avr,
"out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1002 int r = avresample_open(avr);
1005 nb_samples = avresample_convert(avr,
1006 audio_converted->data,
1007 audio_converted->linesize[0],
1008 audio_converted->nb_samples,
1010 audio_frame->linesize[0],
1011 audio_frame->nb_samples);
1014 memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) *
info.
channels);
1017 avresample_close(avr);
1018 avresample_free(&avr);
1022 av_free(audio_converted->data[0]);
1025 int64_t starting_frame_number = -1;
1026 bool partial_frame =
true;
1027 for (
int channel_filter = 0; channel_filter <
info.
channels; channel_filter++)
1030 starting_frame_number = target_frame;
1031 int channel_buffer_size = packet_samples /
info.
channels;
1032 float *channel_buffer =
new float[channel_buffer_size];
1035 for (
int z = 0; z < channel_buffer_size; z++)
1036 channel_buffer[z] = 0.0f;
1042 for (
int sample = 0; sample < packet_samples; sample++)
1045 if (channel_filter == channel)
1048 channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1064 int start = starting_sample;
1065 int remaining_samples = channel_buffer_size;
1066 float *iterate_channel_buffer = channel_buffer;
1067 while (remaining_samples > 0)
1073 int samples = samples_per_frame - start;
1074 if (samples > remaining_samples)
1075 samples = remaining_samples;
1078 std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1081 if (samples_per_frame == start + samples)
1082 partial_frame =
false;
1084 partial_frame =
true;
1088 f->AddAudio(
true, channel_filter, start, iterate_channel_buffer, samples, 0.98f);
1091 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (f->AddAudio)",
"frame", starting_frame_number,
"start", start,
"samples", samples,
"channel", channel_filter,
"partial_frame", partial_frame,
"samples_per_frame", samples_per_frame);
1094 working_cache.
Add(f);
1097 remaining_samples -= samples;
1100 if (remaining_samples > 0)
1101 iterate_channel_buffer += samples;
1104 starting_frame_number++;
1111 delete[] channel_buffer;
1112 channel_buffer = NULL;
1113 iterate_channel_buffer = NULL;
1124 for (int64_t f = target_frame; f < starting_frame_number; f++) {
1128 processing_audio_frames.erase(processing_audio_frames.find(f));
1131 if (processing_audio_frames.count(f) == 0)
1133 processed_audio_frames[f] = f;
1136 if (target_frame == starting_frame_number) {
1138 processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1146 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ProcessAudioPacket (After)",
"requested_frame", requested_frame,
"starting_frame", target_frame,
"end_frame", starting_frame_number - 1,
"", -1,
"", -1,
"", -1);
1153 void FFmpegReader::Seek(int64_t requested_frame)
1156 if (requested_frame < 1)
1157 requested_frame = 1;
1161 int processing_video_frames_size = 0;
1162 int processing_audio_frames_size = 0;
1165 processing_video_frames_size = processing_video_frames.size();
1166 processing_audio_frames_size = processing_audio_frames.size();
1170 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::Seek",
"requested_frame", requested_frame,
"seek_count", seek_count,
"last_frame", last_frame,
"processing_video_frames_size", processing_video_frames_size,
"processing_audio_frames_size", processing_audio_frames_size,
"", -1);
1173 while (processing_video_frames_size + processing_audio_frames_size > 0) {
1176 processing_video_frames_size = processing_video_frames.size();
1177 processing_audio_frames_size = processing_audio_frames.size();
1181 working_cache.
Clear();
1182 missing_frames.
Clear();
1187 processing_audio_frames.clear();
1188 processing_video_frames.clear();
1189 processed_video_frames.clear();
1190 processed_audio_frames.clear();
1191 missing_audio_frames.clear();
1192 missing_video_frames.clear();
1193 missing_audio_frames_source.clear();
1194 missing_video_frames_source.clear();
1195 checked_frames.clear();
1200 current_video_frame = 0;
1201 largest_frame_processed = 0;
1202 num_checks_since_final = 0;
1203 num_packets_since_video_frame = 0;
1204 has_missing_frames =
false;
1212 int buffer_amount = 6;
1213 if (requested_frame - buffer_amount < 20)
1225 if (seek_count == 1) {
1228 seeking_pts = ConvertFrameToVideoPTS(1);
1230 seek_audio_frame_found = 0;
1231 seek_video_frame_found = 0;
1236 bool seek_worked =
false;
1237 int64_t seek_target = 0;
1242 seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1244 fprintf(stderr,
"%s: error while seeking video stream\n", pFormatCtx->filename);
1248 is_video_seek =
true;
1256 seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1258 fprintf(stderr,
"%s: error while seeking audio stream\n", pFormatCtx->filename);
1262 is_video_seek =
false;
1272 avcodec_flush_buffers(aCodecCtx);
1276 avcodec_flush_buffers(pCodecCtx);
1279 previous_packet_location.
frame = -1;
1284 if (seek_count == 1) {
1286 seeking_pts = seek_target;
1287 seeking_frame = requested_frame;
1289 seek_audio_frame_found = 0;
1290 seek_video_frame_found = 0;
1316 int64_t FFmpegReader::GetVideoPTS()
1318 int64_t current_pts = 0;
1319 if(packet->dts != AV_NOPTS_VALUE)
1320 current_pts = packet->dts;
1327 void FFmpegReader::UpdatePTSOffset(
bool is_video)
1333 if (video_pts_offset == 99999)
1339 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::UpdatePTSOffset (Video)",
"video_pts_offset", video_pts_offset,
"is_video", is_video,
"", -1,
"", -1,
"", -1,
"", -1);
1345 if (audio_pts_offset == 99999)
1351 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::UpdatePTSOffset (Audio)",
"audio_pts_offset", audio_pts_offset,
"is_video", is_video,
"", -1,
"", -1,
"", -1,
"", -1);
1357 int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts)
1360 pts = pts + video_pts_offset;
1361 int64_t previous_video_frame = current_video_frame;
1370 if (current_video_frame == 0)
1371 current_video_frame = frame;
1375 if (frame == previous_video_frame) {
1381 current_video_frame++;
1383 if (current_video_frame < frame)
1385 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)",
"calculated frame", frame,
"previous_video_frame", previous_video_frame,
"current_video_frame", current_video_frame,
"", -1,
"", -1,
"", -1);
1390 while (current_video_frame < frame) {
1391 if (!missing_video_frames.count(current_video_frame)) {
1392 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)",
"current_video_frame", current_video_frame,
"previous_video_frame", previous_video_frame,
"", -1,
"", -1,
"", -1,
"", -1);
1393 missing_video_frames.insert(pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1394 missing_video_frames_source.insert(pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1398 has_missing_frames =
true;
1401 current_video_frame++;
1410 int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number)
1419 return video_pts - video_pts_offset;
1423 int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number)
1432 return audio_pts - audio_pts_offset;
1436 AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts)
1439 pts = pts + audio_pts_offset;
1448 int64_t whole_frame = int64_t(frame);
1451 double sample_start_percentage = frame - double(whole_frame);
1457 int sample_start = round(
double(samples_per_frame) * sample_start_percentage);
1460 if (whole_frame < 1)
1462 if (sample_start < 0)
1469 if (previous_packet_location.
frame != -1) {
1470 if (location.
is_near(previous_packet_location, samples_per_frame, samples_per_frame))
1472 int64_t orig_frame = location.
frame;
1477 location.
frame = previous_packet_location.
frame;
1480 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)",
"Source Frame", orig_frame,
"Source Audio Sample", orig_start,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1);
1484 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)",
"Previous location frame", previous_packet_location.
frame,
"Target Frame", location.
frame,
"Target Audio Sample", location.
sample_start,
"pts", pts,
"", -1,
"", -1);
1487 for (int64_t audio_frame = previous_packet_location.
frame; audio_frame < location.
frame; audio_frame++) {
1488 if (!missing_audio_frames.count(audio_frame)) {
1489 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::GetAudioPTSLocation (tracking missing frame)",
"missing_audio_frame", audio_frame,
"previous_audio_frame", previous_packet_location.
frame,
"new location frame", location.
frame,
"", -1,
"", -1,
"", -1);
1490 missing_audio_frames.insert(pair<int64_t, int64_t>(previous_packet_location.
frame - 1, audio_frame));
1497 previous_packet_location = location;
1504 std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame)
1507 std::shared_ptr<Frame> output = working_cache.
GetFrame(requested_frame);
1516 working_cache.
Add(output);
1519 if (requested_frame > largest_frame_processed)
1520 largest_frame_processed = requested_frame;
1528 bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
1531 bool seek_trash =
false;
1532 int64_t max_seeked_frame = seek_audio_frame_found;
1533 if (seek_video_frame_found > max_seeked_frame)
1534 max_seeked_frame = seek_video_frame_found;
1535 if ((
info.
has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
1536 (
info.
has_video && seek_video_frame_found && max_seeked_frame >= requested_frame))
1543 bool FFmpegReader::CheckMissingFrame(int64_t requested_frame)
1549 int checked_count = 0;
1552 if (checked_frames.count(requested_frame) == 0)
1553 checked_frames[requested_frame] = 1;
1555 checked_frames[requested_frame]++;
1556 checked_count = checked_frames[requested_frame];
1559 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame",
"requested_frame", requested_frame,
"has_missing_frames", has_missing_frames,
"missing_video_frames.size()", missing_video_frames.size(),
"checked_count", checked_count,
"", -1,
"", -1);
1563 map<int64_t, int64_t>::iterator itr;
1564 bool found_missing_frame =
false;
1567 if (missing_video_frames.count(requested_frame) || missing_audio_frames.count(requested_frame)) {
1568 int64_t missing_source_frame = -1;
1569 if (missing_video_frames.count(requested_frame))
1570 missing_source_frame = missing_video_frames.find(requested_frame)->second;
1571 else if (missing_audio_frames.count(requested_frame))
1572 missing_source_frame = missing_audio_frames.find(requested_frame)->second;
1575 if (checked_frames.count(missing_source_frame) == 0)
1576 checked_frames[missing_source_frame] = 1;
1578 checked_frames[missing_source_frame]++;
1581 std::shared_ptr<Frame> parent_frame = missing_frames.
GetFrame(missing_source_frame);
1582 if (parent_frame == NULL) {
1584 if (parent_frame != NULL) {
1586 missing_frames.
Add(parent_frame);
1591 std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
1594 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1597 if (parent_frame != NULL) {
1599 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)",
"requested_frame", requested_frame,
"missing_frame->number", missing_frame->number,
"missing_source_frame", missing_source_frame,
"", -1,
"", -1,
"", -1);
1602 std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
1604 missing_frame->AddImage(std::shared_ptr<QImage>(
new QImage(*parent_image)));
1606 processed_video_frames[missing_frame->number] = missing_frame->number;
1607 processed_audio_frames[missing_frame->number] = missing_frame->number;
1613 working_cache.
Remove(missing_frame->number);
1616 last_frame = missing_frame->number;
1622 return found_missing_frame;
1626 void FFmpegReader::CheckWorkingFrames(
bool end_of_stream, int64_t requested_frame)
1629 bool checked_count_tripped =
false;
1630 int max_checked_count = 80;
1643 CheckMissingFrame(f->number);
1646 int checked_count = 0;
1647 int checked_frames_size = 0;
1649 bool is_video_ready =
false;
1650 bool is_audio_ready =
false;
1653 is_video_ready = processed_video_frames.count(f->number);
1654 is_audio_ready = processed_audio_frames.count(f->number);
1657 checked_frames_size = checked_frames.size();
1658 if (!checked_count_tripped || f->number >= requested_frame)
1659 checked_count = checked_frames[f->number];
1662 checked_count = max_checked_count;
1665 if (previous_packet_location.
frame == f->number && !end_of_stream)
1666 is_audio_ready =
false;
1667 bool is_seek_trash = IsPartialFrame(f->number);
1674 if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
1676 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (exceeded checked_count)",
"requested_frame", requested_frame,
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames_size", checked_frames_size);
1679 checked_count_tripped =
true;
1681 if (
info.
has_video && !is_video_ready && last_video_frame) {
1683 f->AddImage(std::shared_ptr<QImage>(
new QImage(*last_video_frame->GetImage())));
1684 is_video_ready =
true;
1690 is_audio_ready =
true;
1695 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames",
"requested_frame", requested_frame,
"frame_number", f->number,
"is_video_ready", is_video_ready,
"is_audio_ready", is_audio_ready,
"checked_count", checked_count,
"checked_frames_size", checked_frames_size);
1698 if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash)
1701 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (mark frame as final)",
"requested_frame", requested_frame,
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1);
1706 num_checks_since_final = 0;
1714 if (missing_video_frames_source.count(f->number)) {
1716 ZmqLogger::Instance()->
AppendDebugMethod(
"FFmpegReader::CheckWorkingFrames (add frame to missing cache)",
"f->number", f->number,
"is_seek_trash", is_seek_trash,
"Missing Cache Count", missing_frames.
Count(),
"Working Cache Count", working_cache.
Count(),
"Final Cache Count",
final_cache.
Count(),
"", -1);
1717 missing_frames.
Add(f);
1721 checked_frames.erase(f->number);
1725 working_cache.
Remove(f->number);
1728 last_frame = f->number;
1732 working_cache.
Remove(f->number);
1742 void FFmpegReader::CheckFPS()
1747 int first_second_counter = 0;
1748 int second_second_counter = 0;
1749 int third_second_counter = 0;
1750 int forth_second_counter = 0;
1751 int fifth_second_counter = 0;
1754 int threshold = 500;
1760 if (GetNextPacket() < 0)
1765 if (packet->stream_index == videoStream)
1771 UpdatePTSOffset(
true);
1774 int64_t pts = GetVideoPTS();
1777 RemoveAVFrame(pFrame);
1780 pts += video_pts_offset;
1786 if (video_seconds <= 1.0)
1787 first_second_counter++;
1788 else if (video_seconds > 1.0 && video_seconds <= 2.0)
1789 second_second_counter++;
1790 else if (video_seconds > 2.0 && video_seconds <= 3.0)
1791 third_second_counter++;
1792 else if (video_seconds > 3.0 && video_seconds <= 4.0)
1793 forth_second_counter++;
1794 else if (video_seconds > 4.0 && video_seconds <= 5.0)
1795 fifth_second_counter++;
1806 if (iterations > threshold)
1811 if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0)
1820 int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
1821 int avg_fps = round(sum_fps / 4.0f);
1829 double diff = fps - double(avg_fps);
1832 if (diff <= -1 || diff >= 1)
1836 diff = half_fps - double(avg_fps);
1839 if (diff <= -1 || diff >= 1)
1856 void FFmpegReader::RemoveAVFrame(AVPicture* remove_frame)
1862 avpicture_free(remove_frame);
1865 delete remove_frame;
1870 void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet)
1876 delete remove_packet;
1880 int64_t FFmpegReader::GetSmallestVideoFrame()
1883 map<int64_t, int64_t>::iterator itr;
1884 int64_t smallest_frame = -1;
1886 for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr)
1888 if (itr->first < smallest_frame || smallest_frame == -1)
1889 smallest_frame = itr->first;
1893 return smallest_frame;
1897 int64_t FFmpegReader::GetSmallestAudioFrame()
1900 map<int64_t, int64_t>::iterator itr;
1901 int64_t smallest_frame = -1;
1903 for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr)
1905 if (itr->first < smallest_frame || smallest_frame == -1)
1906 smallest_frame = itr->first;
1910 return smallest_frame;
1925 root[
"type"] =
"FFmpegReader";
1926 root[
"path"] = path;
1937 Json::Reader reader;
1938 bool success = reader.parse( value, root );
1941 throw InvalidJSON(
"JSON could not be parsed (or is invalid)",
"");
1951 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)",
"");
1962 if (!root[
"path"].isNull())
1963 path = root[
"path"].asString();
#define AV_RESET_FRAME(av_frame)
int max_height
The maximium image height needed by this clip (used for optimizations)
#define AV_FREE_FRAME(av_frame)
int num
Numerator for the fraction.
CriticalSection getFrameCriticalSection
Section lock for multiple threads.
CriticalSection processingCriticalSection
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
int width
The width of the video (in pixesl)
std::shared_ptr< Frame > GetFrame(int64_t requested_frame)
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
float duration
Length of time (in seconds)
string acodec
The name of the audio codec used to encode / decode the video stream.
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define OPEN_MP_NUM_PROCESSORS
string Json()
Get and Set JSON methods.
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Exception when a reader is closed, and a frame is requested.
bool has_video
Determines if this file has a video stream.
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
int64_t file_size
Size of file (in bytes)
FFmpegReader(string path)
std::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
int audio_bit_rate
The bit rate of the audio stream (in bytes)
bool has_audio
Determines if this file has an audio stream.
Exception when no valid codec is found for a file.
int audio_stream_index
The index of the audio stream.
int64_t video_length
The number of frames in the video stream.
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
int height
The height of the video (in pixels)
void SetJson(string value)
Load JSON string into this object.
#define AV_ALLOCATE_FRAME()
Exception for files that can not be found or opened.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
This class represents a fraction.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
ReaderInfo info
Information about the current media file.
void Clear()
Clear the cache of all frames.
int ToInt()
Return a rounded integer of the fraction (for example 30000/1001 returns 30)
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Exception for frames that are out of bounds.
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
int64_t Count()
Count the frames in the queue.
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
This namespace is the default namespace for all code in the openshot library.
Exception for invalid JSON.
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
This struct holds the associated video frame and starting sample # for an audio packet.
void Open()
Open File - which is called by the constructor automatically.
int video_bit_rate
The bit rate of the video stream (in bytes)
Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
string vcodec
The name of the video codec used to encode / decode the video stream.
void Remove(int64_t frame_number)
Remove a specific frame.
CacheMemory final_cache
Final cache object used to hold final frames.
~FFmpegReader()
Destructor.
int den
Denominator for the fraction.
int channels
The number of audio channels used in the audio stream.
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
int video_stream_index
The index of the video stream.
int max_width
The maximum image width needed by this clip (used for optimizations)
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int sample_rate
The number of audio samples per second (44100 is a common sample rate)