OpenShot Library | libopenshot  0.1.9
FFmpegWriter.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for FFmpegWriter class
4  * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2013 OpenShot Studios, LLC, Fabrice Bellard
9  * (http://www.openshotstudios.com). This file is part of
10  * OpenShot Library (http://www.openshot.org), an open-source project
11  * dedicated to delivering high quality video editing and animation solutions
12  * to the world.
13  *
14  * This file is originally based on the Libavformat API example, and then modified
15  * by the libopenshot project.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #include "../include/FFmpegWriter.h"
32 
33 using namespace openshot;
34 
36  path(path), fmt(NULL), oc(NULL), audio_st(NULL), video_st(NULL), audio_pts(0), video_pts(0), samples(NULL),
37  audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0),
38  initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(8), num_of_rescalers(32),
39  rescaler_position(0), video_codec(NULL), audio_codec(NULL), is_writing(false), write_video_count(0), write_audio_count(0),
40  original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false),
41  write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL)
42 {
43 
44  // Disable audio & video (so they can be independently enabled)
45  info.has_audio = false;
46  info.has_video = false;
47 
48  // Initialize FFMpeg, and register all formats and codecs
49  av_register_all();
50 
51  // auto detect format
52  auto_detect_format();
53 }
54 
55 // Open the writer
57 {
58  // Open the writer
59  is_open = true;
60 
61  // Prepare streams (if needed)
62  if (!prepare_streams)
64 
65  // Write header (if needed)
66  if (!write_header)
67  WriteHeader();
68 }
69 
70 // auto detect format (from path)
71 void FFmpegWriter::auto_detect_format()
72 {
73  // Auto detect the output format from the name. default is mpeg.
74  fmt = av_guess_format(NULL, path.c_str(), NULL);
75  if (!fmt)
76  throw InvalidFormat("Could not deduce output format from file extension.", path);
77 
78  // Allocate the output media context
79  oc = avformat_alloc_context();
80  if (!oc)
81  throw OutOfMemory("Could not allocate memory for AVFormatContext.", path);
82 
83  // Set the AVOutputFormat for the current AVFormatContext
84  oc->oformat = fmt;
85 
86  // Update codec names
87  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
88  // Update video codec name
89  info.vcodec = avcodec_find_encoder(fmt->video_codec)->name;
90 
91  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
92  // Update audio codec name
93  info.acodec = avcodec_find_encoder(fmt->audio_codec)->name;
94 }
95 
96 // initialize streams
97 void FFmpegWriter::initialize_streams()
98 {
99  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::initialize_streams", "fmt->video_codec", fmt->video_codec, "fmt->audio_codec", fmt->audio_codec, "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE, "", -1, "", -1, "", -1);
100 
101  // Add the audio and video streams using the default format codecs and initialize the codecs
102  video_st = NULL;
103  audio_st = NULL;
104  if (fmt->video_codec != AV_CODEC_ID_NONE && info.has_video)
105  // Add video stream
106  video_st = add_video_stream();
107 
108  if (fmt->audio_codec != AV_CODEC_ID_NONE && info.has_audio)
109  // Add audio stream
110  audio_st = add_audio_stream();
111 }
112 
113 // Set video export options
114 void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
115 {
116  // Set the video options
117  if (codec.length() > 0)
118  {
119  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
120  if (new_codec == NULL)
121  throw InvalidCodec("A valid video codec could not be found for this file.", path);
122  else {
123  // Set video codec
124  info.vcodec = new_codec->name;
125 
126  // Update video codec in fmt
127  fmt->video_codec = new_codec->id;
128  }
129  }
130  if (fps.num > 0)
131  {
132  // Set frames per second (if provided)
133  info.fps.num = fps.num;
134  info.fps.den = fps.den;
135 
136  // Set the timebase (inverse of fps)
139  }
140  if (width >= 1)
141  info.width = width;
142  if (height >= 1)
143  info.height = height;
144  if (pixel_ratio.num > 0)
145  {
146  info.pixel_ratio.num = pixel_ratio.num;
147  info.pixel_ratio.den = pixel_ratio.den;
148  }
149  if (bit_rate >= 1000)
150  info.video_bit_rate = bit_rate;
151 
152  info.interlaced_frame = interlaced;
153  info.top_field_first = top_field_first;
154 
155  // Calculate the DAR (display aspect ratio)
157 
158  // Reduce size fraction
159  size.Reduce();
160 
161  // Set the ratio based on the reduced fraction
162  info.display_ratio.num = size.num;
163  info.display_ratio.den = size.den;
164 
165  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetVideoOptions (" + codec + ")", "width", width, "height", height, "size.num", size.num, "size.den", size.den, "fps.num", fps.num, "fps.den", fps.den);
166 
167  // Enable / Disable video
168  info.has_video = has_video;
169 }
170 
171 // Set audio export options
172 void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
173 {
174  // Set audio options
175  if (codec.length() > 0)
176  {
177  AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str());
178  if (new_codec == NULL)
179  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
180  else
181  {
182  // Set audio codec
183  info.acodec = new_codec->name;
184 
185  // Update audio codec in fmt
186  fmt->audio_codec = new_codec->id;
187  }
188  }
189  if (sample_rate > 7999)
190  info.sample_rate = sample_rate;
191  if (channels > 0)
192  info.channels = channels;
193  if (bit_rate > 999)
194  info.audio_bit_rate = bit_rate;
195  info.channel_layout = channel_layout;
196 
197  // init resample options (if zero)
198  if (original_sample_rate == 0)
199  original_sample_rate = info.sample_rate;
200  if (original_channels == 0)
201  original_channels = info.channels;
202 
203  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetAudioOptions (" + codec + ")", "sample_rate", sample_rate, "channels", channels, "bit_rate", bit_rate, "", -1, "", -1, "", -1);
204 
205  // Enable / Disable audio
206  info.has_audio = has_audio;
207 }
208 
209 // Set custom options (some codecs accept additional params)
210 void FFmpegWriter::SetOption(StreamType stream, string name, string value)
211 {
212  // Declare codec context
213  AVCodecContext *c = NULL;
214  stringstream convert(value);
215 
216  if (info.has_video && stream == VIDEO_STREAM && video_st)
217  c = video_st->codec;
218  else if (info.has_audio && stream == AUDIO_STREAM && audio_st)
219  c = audio_st->codec;
220  else
221  throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path);
222 
223  // Init AVOption
224  const AVOption *option = NULL;
225 
226  // Was a codec / stream found?
227  if (c)
228  // Find AVOption (if it exists)
229  #if LIBAVFORMAT_VERSION_MAJOR <= 53
230  option = av_find_opt(c->priv_data, name.c_str(), NULL, NULL, NULL);
231  #else
232  option = av_opt_find(c->priv_data, name.c_str(), NULL, 0, 0);
233  #endif
234 
235  // Was option found?
236  if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" ||
237  name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate"))
238  {
239  // Check for specific named options
240  if (name == "g")
241  // Set gop_size
242  convert >> c->gop_size;
243 
244  else if (name == "qmin")
245  // Minimum quantizer
246  convert >> c->qmin;
247 
248  else if (name == "qmax")
249  // Maximum quantizer
250  convert >> c->qmax;
251 
252  else if (name == "max_b_frames")
253  // Maximum number of B-frames between non-B-frames
254  convert >> c->max_b_frames;
255 
256  else if (name == "mb_decision")
257  // Macroblock decision mode
258  convert >> c->mb_decision;
259 
260  else if (name == "level")
261  // Set codec level
262  convert >> c->level;
263 
264  else if (name == "profile")
265  // Set codec profile
266  convert >> c->profile;
267 
268  else if (name == "slices")
269  // Indicates number of picture subdivisions
270  convert >> c->slices;
271 
272  else if (name == "rc_min_rate")
273  // Minimum bitrate
274  convert >> c->rc_min_rate;
275 
276  else if (name == "rc_max_rate")
277  // Maximum bitrate
278  convert >> c->rc_max_rate;
279 
280  else if (name == "rc_buffer_size")
281  // Buffer size
282  convert >> c->rc_buffer_size;
283 
284  else
285  // Set AVOption
286  #if LIBAVFORMAT_VERSION_MAJOR <= 53
287  av_set_string3 (c->priv_data, name.c_str(), value.c_str(), 0, NULL);
288  #else
289  av_opt_set (c->priv_data, name.c_str(), value.c_str(), 0);
290  #endif
291 
292  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetOption (" + (string)name + ")", "stream == VIDEO_STREAM", stream == VIDEO_STREAM, "", -1, "", -1, "", -1, "", -1, "", -1);
293 
294  }
295  else
296  throw InvalidOptions("The option is not valid for this codec.", path);
297 
298 }
299 
300 // Prepare & initialize streams and open codecs
302 {
303  if (!info.has_audio && !info.has_video)
304  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
305 
306  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::PrepareStreams [" + path + "]", "info.has_audio", info.has_audio, "info.has_video", info.has_video, "", -1, "", -1, "", -1, "", -1);
307 
308  // Initialize the streams (i.e. add the streams)
309  initialize_streams();
310 
311  // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
312  if (info.has_video && video_st)
313  open_video(oc, video_st);
314  if (info.has_audio && audio_st)
315  open_audio(oc, audio_st);
316 
317  // Mark as 'prepared'
318  prepare_streams = true;
319 }
320 
321 // Write the file header (after the options are set)
323 {
324  if (!info.has_audio && !info.has_video)
325  throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path);
326 
327  // Open the output file, if needed
328  if (!(fmt->flags & AVFMT_NOFILE)) {
329  if (avio_open(&oc->pb, path.c_str(), AVIO_FLAG_WRITE) < 0)
330  throw InvalidFile("Could not open or write file.", path);
331  }
332 
333  // Force the output filename (which doesn't always happen for some reason)
334  snprintf(oc->filename, sizeof(oc->filename), "%s", path.c_str());
335 
336  // Write the stream header, if any
337  // TODO: add avoptions / parameters instead of NULL
338  avformat_write_header(oc, NULL);
339 
340  // Mark as 'written'
341  write_header = true;
342 
343  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteHeader", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
344 }
345 
346 // Add a frame to the queue waiting to be encoded.
347 void FFmpegWriter::WriteFrame(std::shared_ptr<Frame> frame)
348 {
349  // Check for open reader (or throw exception)
350  if (!is_open)
351  throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path);
352 
353  // Add frame pointer to "queue", waiting to be processed the next
354  // time the WriteFrames() method is called.
355  if (info.has_video && video_st)
356  spooled_video_frames.push_back(frame);
357 
358  if (info.has_audio && audio_st)
359  spooled_audio_frames.push_back(frame);
360 
361  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame", "frame->number", frame->number, "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "cache_size", cache_size, "is_writing", is_writing, "", -1);
362 
363  // Write the frames once it reaches the correct cache size
364  if (spooled_video_frames.size() == cache_size || spooled_audio_frames.size() == cache_size)
365  {
366  // Is writer currently writing?
367  if (!is_writing)
368  // Write frames to video file
369  write_queued_frames();
370 
371  else
372  {
373  // YES, WRITING... so wait until it finishes, before writing again
374  while (is_writing)
375  usleep(250000); // sleep for 250 milliseconds
376 
377  // Write frames to video file
378  write_queued_frames();
379  }
380  }
381 
382  // Keep track of the last frame added
383  last_frame = frame;
384 }
385 
386 // Write all frames in the queue to the video file.
387 void FFmpegWriter::write_queued_frames()
388 {
389  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_queued_frames", "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "", -1, "", -1, "", -1, "", -1);
390 
391  // Flip writing flag
392  is_writing = true;
393 
394  // Transfer spool to queue
395  queued_video_frames = spooled_video_frames;
396  queued_audio_frames = spooled_audio_frames;
397 
398  // Empty spool
399  spooled_video_frames.clear();
400  spooled_audio_frames.clear();
401 
402  // Set the number of threads in OpenMP
403  omp_set_num_threads(OPEN_MP_NUM_PROCESSORS);
404  // Allow nested OpenMP sections
405  omp_set_nested(true);
406 
407  // Create blank exception
408  bool has_error_encoding_video = false;
409 
410  #pragma omp parallel
411  {
412  #pragma omp single
413  {
414  // Process all audio frames (in a separate thread)
415  if (info.has_audio && audio_st && !queued_audio_frames.empty())
416  write_audio_packets(false);
417 
418  // Loop through each queued image frame
419  while (!queued_video_frames.empty())
420  {
421  // Get front frame (from the queue)
422  std::shared_ptr<Frame> frame = queued_video_frames.front();
423 
424  // Add to processed queue
425  processed_frames.push_back(frame);
426 
427  // Encode and add the frame to the output file
428  if (info.has_video && video_st)
429  process_video_packet(frame);
430 
431  // Remove front item
432  queued_video_frames.pop_front();
433 
434  } // end while
435  } // end omp single
436 
437  #pragma omp single
438  {
439  // Loop back through the frames (in order), and write them to the video file
440  while (!processed_frames.empty())
441  {
442  // Get front frame (from the queue)
443  std::shared_ptr<Frame> frame = processed_frames.front();
444 
445  if (info.has_video && video_st)
446  {
447  // Add to deallocate queue (so we can remove the AVFrames when we are done)
448  deallocate_frames.push_back(frame);
449 
450  // Does this frame's AVFrame still exist
451  if (av_frames.count(frame))
452  {
453  // Get AVFrame
454  AVFrame *frame_final = av_frames[frame];
455 
456  // Write frame to video file
457  bool success = write_video_packet(frame, frame_final);
458  if (!success)
459  has_error_encoding_video = true;
460  }
461  }
462 
463  // Remove front item
464  processed_frames.pop_front();
465  }
466 
467  // Loop through, and deallocate AVFrames
468  while (!deallocate_frames.empty())
469  {
470  // Get front frame (from the queue)
471  std::shared_ptr<Frame> frame = deallocate_frames.front();
472 
473  // Does this frame's AVFrame still exist
474  if (av_frames.count(frame))
475  {
476  // Get AVFrame
477  AVFrame *av_frame = av_frames[frame];
478 
479  // Deallocate AVPicture and AVFrame
480  av_freep(&(av_frame->data[0]));
481  AV_FREE_FRAME(&av_frame);
482  av_frames.erase(frame);
483  }
484 
485  // Remove front item
486  deallocate_frames.pop_front();
487  }
488 
489  // Done writing
490  is_writing = false;
491 
492  } // end omp single
493  } // end omp parallel
494 
495  // Raise exception from main thread
496  if (has_error_encoding_video)
497  throw ErrorEncodingVideo("Error while writing raw video frame", -1);
498 }
499 
500 // Write a block of frames from a reader
501 void FFmpegWriter::WriteFrame(ReaderBase* reader, int64_t start, int64_t length)
502 {
503  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame (from Reader)", "start", start, "length", length, "", -1, "", -1, "", -1, "", -1);
504 
505  // Loop through each frame (and encoded it)
506  for (int64_t number = start; number <= length; number++)
507  {
508  // Get the frame
509  std::shared_ptr<Frame> f = reader->GetFrame(number);
510 
511  // Encode frame
512  WriteFrame(f);
513  }
514 }
515 
516 // Write the file trailer (after all frames are written)
518 {
519  // Write any remaining queued frames to video file
520  write_queued_frames();
521 
522  // Process final audio frame (if any)
523  if (info.has_audio && audio_st)
524  write_audio_packets(true);
525 
526  // Flush encoders (who sometimes hold on to frames)
527  flush_encoders();
528 
529  /* write the trailer, if any. The trailer must be written
530  * before you close the CodecContexts open when you wrote the
531  * header; otherwise write_trailer may try to use memory that
532  * was freed on av_codec_close() */
533  av_write_trailer(oc);
534 
535  // Mark as 'written'
536  write_trailer = true;
537 
538  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteTrailer", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
539 }
540 
541 // Flush encoders
542 void FFmpegWriter::flush_encoders()
543 {
544  if (info.has_audio && audio_codec && audio_st->codec->codec_type == AVMEDIA_TYPE_AUDIO && audio_codec->frame_size <= 1)
545  return;
546  if (info.has_video && video_st->codec->codec_type == AVMEDIA_TYPE_VIDEO && (oc->oformat->flags & AVFMT_RAWPICTURE) && video_codec->codec->id == AV_CODEC_ID_RAWVIDEO)
547  return;
548 
549  int error_code = 0;
550  int stop_encoding = 1;
551 
552  // FLUSH VIDEO ENCODER
553  if (info.has_video)
554  for (;;) {
555 
556  // Increment PTS (in frames and scaled to the codec's timebase)
557  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
558 
559  AVPacket pkt;
560  av_init_packet(&pkt);
561  pkt.data = NULL;
562  pkt.size = 0;
563 
564  // Pointer for video buffer (if using old FFmpeg version)
565  uint8_t *video_outbuf = NULL;
566 
567  /* encode the image */
568  int got_packet = 0;
569  int error_code = 0;
570 
571  #if LIBAVFORMAT_VERSION_MAJOR >= 54
572  // Newer versions of FFMpeg
573  error_code = avcodec_encode_video2(video_codec, &pkt, NULL, &got_packet);
574 
575  #else
576  // Older versions of FFmpeg (much sloppier)
577 
578  // Encode Picture and Write Frame
579  int video_outbuf_size = 0;
580 
581  /* encode the image */
582  int out_size = avcodec_encode_video(video_codec, NULL, video_outbuf_size, NULL);
583 
584  /* if zero size, it means the image was buffered */
585  if (out_size > 0) {
586  if(video_codec->coded_frame->key_frame)
587  pkt.flags |= AV_PKT_FLAG_KEY;
588  pkt.data= video_outbuf;
589  pkt.size= out_size;
590 
591  // got data back (so encode this frame)
592  got_packet = 1;
593  }
594  #endif
595 
596  if (error_code < 0) {
597  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
598  }
599  if (!got_packet) {
600  stop_encoding = 1;
601  break;
602  }
603 
604  // Override PTS (in frames and scaled to the codec's timebase)
605  //pkt.pts = write_video_count;
606 
607  // set the timestamp
608  if (pkt.pts != AV_NOPTS_VALUE)
609  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
610  if (pkt.dts != AV_NOPTS_VALUE)
611  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
612  if (pkt.duration > 0)
613  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
614  pkt.stream_index = video_st->index;
615 
616  // Write packet
617  error_code = av_interleaved_write_frame(oc, &pkt);
618  if (error_code < 0) {
619  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
620  }
621 
622  // Deallocate memory (if needed)
623  if (video_outbuf)
624  av_freep(&video_outbuf);
625  }
626 
627  // FLUSH AUDIO ENCODER
628  if (info.has_audio)
629  for (;;) {
630 
631  // Increment PTS (in samples and scaled to the codec's timebase)
632 #if LIBAVFORMAT_VERSION_MAJOR >= 54
633  // for some reason, it requires me to multiply channels X 2
634  write_audio_count += av_rescale_q(audio_input_position / (audio_codec->channels * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)), (AVRational){1, info.sample_rate}, audio_codec->time_base);
635 #else
636  write_audio_count += av_rescale_q(audio_input_position / audio_codec->channels, (AVRational){1, info.sample_rate}, audio_codec->time_base);
637 #endif
638 
639  AVPacket pkt;
640  av_init_packet(&pkt);
641  pkt.data = NULL;
642  pkt.size = 0;
643  pkt.pts = pkt.dts = write_audio_count;
644 
645  /* encode the image */
646  int got_packet = 0;
647  error_code = avcodec_encode_audio2(audio_codec, &pkt, NULL, &got_packet);
648  if (error_code < 0) {
649  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
650  }
651  if (!got_packet) {
652  stop_encoding = 1;
653  break;
654  }
655 
656  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
657  // but it fixes lots of PTS related issues when I do this.
658  pkt.pts = pkt.dts = write_audio_count;
659 
660  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
661  if (pkt.pts != AV_NOPTS_VALUE)
662  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
663  if (pkt.dts != AV_NOPTS_VALUE)
664  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
665  if (pkt.duration > 0)
666  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
667 
668  // set stream
669  pkt.stream_index = audio_st->index;
670  pkt.flags |= AV_PKT_FLAG_KEY;
671 
672  // Write packet
673  error_code = av_interleaved_write_frame(oc, &pkt);
674  if (error_code < 0) {
675  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
676  }
677 
678  // deallocate memory for packet
679  AV_FREE_PACKET(&pkt);
680  }
681 
682 
683 }
684 
685 // Close the video codec
686 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
687 {
688  avcodec_close(st->codec);
689  video_codec = NULL;
690 }
691 
692 // Close the audio codec
693 void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st)
694 {
695  avcodec_close(st->codec);
696  audio_codec = NULL;
697 
698  // Clear buffers
699  delete[] samples;
700  delete[] audio_outbuf;
701  delete[] audio_encoder_buffer;
702  samples = NULL;
703  audio_outbuf = NULL;
704  audio_encoder_buffer = NULL;
705 
706  // Deallocate resample buffer
707  if (avr) {
708  avresample_close(avr);
709  avresample_free(&avr);
710  avr = NULL;
711  }
712 
713  if (avr_planar) {
714  avresample_close(avr_planar);
715  avresample_free(&avr_planar);
716  avr_planar = NULL;
717  }
718 }
719 
720 // Close the writer
722 {
723  // Write trailer (if needed)
724  if (!write_trailer)
725  WriteTrailer();
726 
727  // Close each codec
728  if (video_st)
729  close_video(oc, video_st);
730  if (audio_st)
731  close_audio(oc, audio_st);
732 
733  // Deallocate image scalers
734  if (image_rescalers.size() > 0)
735  RemoveScalers();
736 
737  // Free the streams
738  for (int i = 0; i < oc->nb_streams; i++) {
739  av_freep(&oc->streams[i]->codec);
740  av_freep(&oc->streams[i]);
741  }
742 
743  if (!(fmt->flags & AVFMT_NOFILE)) {
744  /* close the output file */
745  avio_close(oc->pb);
746  }
747 
748  // Reset frame counters
749  write_video_count = 0;
750  write_audio_count = 0;
751 
752  // Free the context
753  av_freep(&oc);
754 
755  // Close writer
756  is_open = false;
757  prepare_streams = false;
758  write_header = false;
759  write_trailer = false;
760 
761  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1);
762 }
763 
764 // Add an AVFrame to the cache
765 void FFmpegWriter::add_avframe(std::shared_ptr<Frame> frame, AVFrame* av_frame)
766 {
767  // Add AVFrame to map (if it does not already exist)
768  if (!av_frames.count(frame))
769  {
770  // Add av_frame
771  av_frames[frame] = av_frame;
772  }
773  else
774  {
775  // Do not add, and deallocate this AVFrame
776  AV_FREE_FRAME(&av_frame);
777  }
778 }
779 
780 // Add an audio output stream
781 AVStream* FFmpegWriter::add_audio_stream()
782 {
783  AVCodecContext *c;
784  AVStream *st;
785 
786  // Find the audio codec
787  AVCodec *codec = avcodec_find_encoder_by_name(info.acodec.c_str());
788  if (codec == NULL)
789  throw InvalidCodec("A valid audio codec could not be found for this file.", path);
790 
791  // Create a new audio stream
792  st = avformat_new_stream(oc, codec);
793  if (!st)
794  throw OutOfMemory("Could not allocate memory for the audio stream.", path);
795 
796  // Set default values
797  avcodec_get_context_defaults3(st->codec, codec);
798 
799  c = st->codec;
800  c->codec_id = codec->id;
801 #if LIBAVFORMAT_VERSION_MAJOR >= 53
802  c->codec_type = AVMEDIA_TYPE_AUDIO;
803 #else
804  c->codec_type = CODEC_TYPE_AUDIO;
805 #endif
806 
807  // Set the sample parameters
808  c->bit_rate = info.audio_bit_rate;
809  c->channels = info.channels;
810 
811  // Set valid sample rate (or throw error)
812  if (codec->supported_samplerates) {
813  int i;
814  for (i = 0; codec->supported_samplerates[i] != 0; i++)
815  if (info.sample_rate == codec->supported_samplerates[i])
816  {
817  // Set the valid sample rate
818  c->sample_rate = info.sample_rate;
819  break;
820  }
821  if (codec->supported_samplerates[i] == 0)
822  throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path);
823  } else
824  // Set sample rate
825  c->sample_rate = info.sample_rate;
826 
827 
828  // Set a valid number of channels (or throw error)
829  int channel_layout = info.channel_layout;
830  if (codec->channel_layouts) {
831  int i;
832  for (i = 0; codec->channel_layouts[i] != 0; i++)
833  if (channel_layout == codec->channel_layouts[i])
834  {
835  // Set valid channel layout
836  c->channel_layout = channel_layout;
837  break;
838  }
839  if (codec->channel_layouts[i] == 0)
840  throw InvalidChannels("An invalid channel layout was detected (i.e. MONO / STEREO).", path);
841  } else
842  // Set valid channel layout
843  c->channel_layout = channel_layout;
844 
845  // Choose a valid sample_fmt
846  if (codec->sample_fmts) {
847  for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++)
848  {
849  // Set sample format to 1st valid format (and then exit loop)
850  c->sample_fmt = codec->sample_fmts[i];
851  break;
852  }
853  }
854  if (c->sample_fmt == AV_SAMPLE_FMT_NONE) {
855  // Default if no sample formats found
856  c->sample_fmt = AV_SAMPLE_FMT_S16;
857  }
858 
859  // some formats want stream headers to be separate
860  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
861  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
862 
863  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_audio_stream", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->channels", c->channels, "c->sample_fmt", c->sample_fmt, "c->channel_layout", c->channel_layout, "c->sample_rate", c->sample_rate);
864 
865  return st;
866 }
867 
868 // Add a video output stream
869 AVStream* FFmpegWriter::add_video_stream()
870 {
871  AVCodecContext *c;
872  AVStream *st;
873 
874  // Find the audio codec
875  AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
876  if (codec == NULL)
877  throw InvalidCodec("A valid video codec could not be found for this file.", path);
878 
879  // Create a new stream
880  st = avformat_new_stream(oc, codec);
881  if (!st)
882  throw OutOfMemory("Could not allocate memory for the video stream.", path);
883 
884  // Set default values
885  avcodec_get_context_defaults3(st->codec, codec);
886 
887  c = st->codec;
888  c->codec_id = codec->id;
889 #if LIBAVFORMAT_VERSION_MAJOR >= 53
890  c->codec_type = AVMEDIA_TYPE_VIDEO;
891 #else
892  c->codec_type = CODEC_TYPE_VIDEO;
893 #endif
894 
895  /* Init video encoder options */
896  c->bit_rate = info.video_bit_rate;
897 
898  //TODO: Implement variable bitrate feature (which actually works). This implementation throws
899  //invalid bitrate errors and rc buffer underflow errors, etc...
900  //c->rc_min_rate = info.video_bit_rate;
901  //c->rc_max_rate = info.video_bit_rate;
902  //c->rc_buffer_size = FFMAX(c->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
903  //if ( !c->rc_initial_buffer_occupancy )
904  // c->rc_initial_buffer_occupancy = c->rc_buffer_size * 3/4;
905  c->qmin = 2;
906  c->qmax = 30;
907 
908  /* resolution must be a multiple of two */
909  // TODO: require /2 height and width
910  c->width = info.width;
911  c->height = info.height;
912 
913  /* time base: this is the fundamental unit of time (in seconds) in terms
914  of which frame timestamps are represented. for fixed-fps content,
915  timebase should be 1/framerate and timestamp increments should be
916  identically 1. */
917  c->time_base.num = info.video_timebase.num;
918  c->time_base.den = info.video_timebase.den;
919  st->time_base.num = info.video_timebase.num;
920  st->time_base.den = info.video_timebase.den;
921 
922  c->gop_size = 12; /* TODO: add this to "info"... emit one intra frame every twelve frames at most */
923  c->max_b_frames = 10;
924  if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
925  /* just for testing, we also add B frames */
926  c->max_b_frames = 2;
927  if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
928  /* Needed to avoid using macroblocks in which some coeffs overflow.
929  This does not happen with normal video, it just happens here as
930  the motion of the chroma plane does not match the luma plane. */
931  c->mb_decision = 2;
932  // some formats want stream headers to be separate
933  if (oc->oformat->flags & AVFMT_GLOBALHEADER)
934  c->flags |= CODEC_FLAG_GLOBAL_HEADER;
935 
936  // Find all supported pixel formats for this codec
937  const PixelFormat* supported_pixel_formats = codec->pix_fmts;
938  while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) {
939  // Assign the 1st valid pixel format (if one is missing)
940  if (c->pix_fmt == PIX_FMT_NONE)
941  c->pix_fmt = *supported_pixel_formats;
942  ++supported_pixel_formats;
943  }
944 
945  // Codec doesn't have any pix formats?
946  if (c->pix_fmt == PIX_FMT_NONE) {
947  if(fmt->video_codec == AV_CODEC_ID_RAWVIDEO) {
948  // Raw video should use RGB24
949  c->pix_fmt = PIX_FMT_RGB24;
950 
951  if (strcmp(fmt->name, "gif") != 0)
952  // If not GIF format, skip the encoding process
953  // Set raw picture flag (so we don't encode this video)
954  oc->oformat->flags |= AVFMT_RAWPICTURE;
955  } else {
956  // Set the default codec
957  c->pix_fmt = PIX_FMT_YUV420P;
958  }
959  }
960 
961  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::add_video_stream (" + (string)fmt->name + " : " + (string)av_get_pix_fmt_name(c->pix_fmt) + ")", "c->codec_id", c->codec_id, "c->bit_rate", c->bit_rate, "c->pix_fmt", c->pix_fmt, "oc->oformat->flags", oc->oformat->flags, "AVFMT_RAWPICTURE", AVFMT_RAWPICTURE, "", -1);
962 
963  return st;
964 }
965 
966 // open audio codec
967 void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st)
968 {
969  AVCodec *codec;
970  audio_codec = st->codec;
971 
972  // Set number of threads equal to number of processors (not to exceed 16)
973  audio_codec->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
974 
975  // Find the audio encoder
976  codec = avcodec_find_encoder_by_name(info.acodec.c_str());
977  if (!codec)
978  codec = avcodec_find_encoder(audio_codec->codec_id);
979  if (!codec)
980  throw InvalidCodec("Could not find codec", path);
981 
982  // Open the codec
983  if (avcodec_open2(audio_codec, codec, NULL) < 0)
984  throw InvalidCodec("Could not open codec", path);
985 
986  // Calculate the size of the input frame (i..e how many samples per packet), and the output buffer
987  // TODO: Ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples
988  if (audio_codec->frame_size <= 1) {
989  // No frame size found... so calculate
990  audio_input_frame_size = 50000 / info.channels;
991 
992  switch (st->codec->codec_id) {
993  case AV_CODEC_ID_PCM_S16LE:
994  case AV_CODEC_ID_PCM_S16BE:
995  case AV_CODEC_ID_PCM_U16LE:
996  case AV_CODEC_ID_PCM_U16BE:
997  audio_input_frame_size >>= 1;
998  break;
999  default:
1000  break;
1001  }
1002  } else {
1003  // Set frame size based on the codec
1004  audio_input_frame_size = audio_codec->frame_size;
1005  }
1006 
1007  // Set the initial frame size (since it might change during resampling)
1008  initial_audio_input_frame_size = audio_input_frame_size;
1009 
1010  // Allocate array for samples
1011  samples = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE];
1012 
1013  // Set audio output buffer (used to store the encoded audio)
1014  audio_outbuf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1015  audio_outbuf = new uint8_t[audio_outbuf_size];
1016 
1017  // Set audio packet encoding buffer
1018  audio_encoder_buffer_size = AUDIO_PACKET_ENCODING_SIZE;
1019  audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size];
1020 
1021  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_audio", "audio_codec->thread_count", audio_codec->thread_count, "audio_input_frame_size", audio_input_frame_size, "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE, "", -1, "", -1, "", -1);
1022 
1023 }
1024 
1025 // open video codec
1026 void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st)
1027 {
1028  AVCodec *codec;
1029  video_codec = st->codec;
1030 
1031  // Set number of threads equal to number of processors (not to exceed 16)
1032  video_codec->thread_count = min(OPEN_MP_NUM_PROCESSORS, 16);
1033 
1034  /* find the video encoder */
1035  codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
1036  if (!codec)
1037  codec = avcodec_find_encoder(video_codec->codec_id);
1038  if (!codec)
1039  throw InvalidCodec("Could not find codec", path);
1040 
1041  /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */
1042  if(video_codec->max_b_frames && video_codec->codec_id != AV_CODEC_ID_MPEG4 && video_codec->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec->codec_id != AV_CODEC_ID_MPEG2VIDEO)
1043  video_codec->max_b_frames = 0;
1044 
1045  /* open the codec */
1046  if (avcodec_open2(video_codec, codec, NULL) < 0)
1047  throw InvalidCodec("Could not open codec", path);
1048 
1049  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video", "video_codec->thread_count", video_codec->thread_count, "", -1, "", -1, "", -1, "", -1, "", -1);
1050 
1051 }
1052 
1053 // write all queued frames' audio to the video file
1054 void FFmpegWriter::write_audio_packets(bool final)
1055 {
1056  #pragma omp task firstprivate(final)
1057  {
1058  // Init audio buffers / variables
1059  int total_frame_samples = 0;
1060  int frame_position = 0;
1061  int channels_in_frame = 0;
1062  int sample_rate_in_frame = 0;
1063  int samples_in_frame = 0;
1064  ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout
1065 
1066  // Create a new array (to hold all S16 audio samples, for the current queued frames
1067  int16_t* all_queued_samples = (int16_t*)av_malloc((sizeof(int16_t)*(queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE)));
1068  int16_t* all_resampled_samples = NULL;
1069  int16_t* final_samples_planar = NULL;
1070  int16_t* final_samples = NULL;
1071 
1072  // Loop through each queued audio frame
1073  while (!queued_audio_frames.empty())
1074  {
1075  // Get front frame (from the queue)
1076  std::shared_ptr<Frame> frame = queued_audio_frames.front();
1077 
1078  // Get the audio details from this frame
1079  sample_rate_in_frame = frame->SampleRate();
1080  samples_in_frame = frame->GetAudioSamplesCount();
1081  channels_in_frame = frame->GetAudioChannelsCount();
1082  channel_layout_in_frame = frame->ChannelsLayout();
1083 
1084 
1085  // Get audio sample array
1086  float* frame_samples_float = NULL;
1087  // Get samples interleaved together (c1 c2 c1 c2 c1 c2)
1088  frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame);
1089 
1090 
1091  // Calculate total samples
1092  total_frame_samples = samples_in_frame * channels_in_frame;
1093 
1094  // Translate audio sample values back to 16 bit integers
1095  for (int s = 0; s < total_frame_samples; s++, frame_position++)
1096  // Translate sample value and copy into buffer
1097  all_queued_samples[frame_position] = int(frame_samples_float[s] * (1 << 15));
1098 
1099 
1100  // Deallocate float array
1101  delete[] frame_samples_float;
1102 
1103  // Remove front item
1104  queued_audio_frames.pop_front();
1105 
1106  } // end while
1107 
1108 
1109  // Update total samples (since we've combined all queued frames)
1110  total_frame_samples = frame_position;
1111  int remaining_frame_samples = total_frame_samples;
1112  int samples_position = 0;
1113 
1114 
1115  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets", "final", final, "total_frame_samples", total_frame_samples, "channel_layout_in_frame", channel_layout_in_frame, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "LAYOUT_MONO", LAYOUT_MONO);
1116 
1117  // Keep track of the original sample format
1118  AVSampleFormat output_sample_fmt = audio_codec->sample_fmt;
1119 
1120  AVFrame *audio_frame = NULL;
1121  if (!final) {
1122  // Create input frame (and allocate arrays)
1123  audio_frame = AV_ALLOCATE_FRAME();
1124  AV_RESET_FRAME(audio_frame);
1125  audio_frame->nb_samples = total_frame_samples / channels_in_frame;
1126 
1127  // Fill input frame with sample data
1128  avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples,
1129  audio_encoder_buffer_size, 0);
1130 
1131  // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point.
1132  switch (audio_codec->sample_fmt)
1133  {
1134  case AV_SAMPLE_FMT_FLTP:
1135  {
1136  output_sample_fmt = AV_SAMPLE_FMT_FLT;
1137  break;
1138  }
1139  case AV_SAMPLE_FMT_S32P:
1140  {
1141  output_sample_fmt = AV_SAMPLE_FMT_S32;
1142  break;
1143  }
1144  case AV_SAMPLE_FMT_S16P:
1145  {
1146  output_sample_fmt = AV_SAMPLE_FMT_S16;
1147  break;
1148  }
1149  case AV_SAMPLE_FMT_U8P:
1150  {
1151  output_sample_fmt = AV_SAMPLE_FMT_U8;
1152  break;
1153  }
1154  }
1155 
1156  // Update total samples & input frame size (due to bigger or smaller data types)
1157  total_frame_samples *= (float(info.sample_rate) / sample_rate_in_frame); // adjust for different byte sizes
1158  total_frame_samples *= (float(info.channels) / channels_in_frame); // adjust for different # of channels
1159 
1160  // Set remaining samples
1161  remaining_frame_samples = total_frame_samples;
1162 
1163  // Create output frame (and allocate arrays)
1164  AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1165  AV_RESET_FRAME(audio_converted);
1166  audio_converted->nb_samples = total_frame_samples / channels_in_frame;
1167  av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_converted->nb_samples, output_sample_fmt, 0);
1168 
1169  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (1st resampling)", "in_sample_fmt", AV_SAMPLE_FMT_S16, "out_sample_fmt", output_sample_fmt, "in_sample_rate", sample_rate_in_frame, "out_sample_rate", info.sample_rate, "in_channels", channels_in_frame, "out_channels", info.channels);
1170 
1171  // setup resample context
1172  if (!avr) {
1173  avr = avresample_alloc_context();
1174  av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0);
1175  av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0);
1176  av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1177  av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here
1178  av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0);
1179  av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1180  av_opt_set_int(avr, "in_channels", channels_in_frame, 0);
1181  av_opt_set_int(avr, "out_channels", info.channels, 0);
1182  avresample_open(avr);
1183  }
1184  int nb_samples = 0;
1185 
1186  // Convert audio samples
1187  nb_samples = avresample_convert(avr, // audio resample context
1188  audio_converted->data, // output data pointers
1189  audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1190  audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1191  audio_frame->data, // input data pointers
1192  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1193  audio_frame->nb_samples); // number of input samples to convert
1194 
1195  // Create a new array (to hold all resampled S16 audio samples)
1196  all_resampled_samples = (int16_t*)av_malloc(sizeof(int16_t) * nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1197 
1198  // Copy audio samples over original samples
1199  memcpy(all_resampled_samples, audio_converted->data[0], nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1200 
1201  // Remove converted audio
1202  av_freep(&(audio_frame->data[0]));
1203  AV_FREE_FRAME(&audio_frame);
1204  av_freep(&audio_converted->data[0]);
1205  AV_FREE_FRAME(&audio_converted);
1206  all_queued_samples = NULL; // this array cleared with above call
1207 
1208  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)", "nb_samples", nb_samples, "remaining_frame_samples", remaining_frame_samples, "", -1, "", -1, "", -1, "", -1);
1209  }
1210 
1211  // Loop until no more samples
1212  while (remaining_frame_samples > 0 || final) {
1213  // Get remaining samples needed for this packet
1214  int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position;
1215 
1216  // Determine how many samples we need
1217  int diff = 0;
1218  if (remaining_frame_samples >= remaining_packet_samples)
1219  diff = remaining_packet_samples;
1220  else if (remaining_frame_samples < remaining_packet_samples)
1221  diff = remaining_frame_samples;
1222 
1223  // Copy frame samples into the packet samples array
1224  if (!final)
1225  //TODO: Make this more sane
1226  memcpy(samples + (audio_input_position * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))), all_resampled_samples + samples_position, diff * av_get_bytes_per_sample(output_sample_fmt));
1227 
1228  // Increment counters
1229  audio_input_position += diff;
1230  samples_position += diff * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16));
1231  remaining_frame_samples -= diff;
1232  remaining_packet_samples -= diff;
1233 
1234  // Do we have enough samples to proceed?
1235  if (audio_input_position < (audio_input_frame_size * info.channels) && !final)
1236  // Not enough samples to encode... so wait until the next frame
1237  break;
1238 
1239  // Convert to planar (if needed by audio codec)
1240  AVFrame *frame_final = AV_ALLOCATE_FRAME();
1241  AV_RESET_FRAME(frame_final);
1242  if (av_sample_fmt_is_planar(audio_codec->sample_fmt))
1243  {
1244  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)", "in_sample_fmt", output_sample_fmt, "out_sample_fmt", audio_codec->sample_fmt, "in_sample_rate", info.sample_rate, "out_sample_rate", info.sample_rate, "in_channels", info.channels, "out_channels", info.channels);
1245 
1246  // setup resample context
1247  if (!avr_planar) {
1248  avr_planar = avresample_alloc_context();
1249  av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0);
1250  av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0);
1251  av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0);
1252  av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec->sample_fmt, 0); // planar not allowed here
1253  av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0);
1254  av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0);
1255  av_opt_set_int(avr_planar, "in_channels", info.channels, 0);
1256  av_opt_set_int(avr_planar, "out_channels", info.channels, 0);
1257  avresample_open(avr_planar);
1258  }
1259 
1260  // Create input frame (and allocate arrays)
1261  audio_frame = AV_ALLOCATE_FRAME();
1262  AV_RESET_FRAME(audio_frame);
1263  audio_frame->nb_samples = audio_input_position / info.channels;
1264 
1265  // Create a new array
1266  final_samples_planar = (int16_t*)av_malloc(sizeof(int16_t) * audio_frame->nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16)));
1267 
1268  // Copy audio into buffer for frame
1269  memcpy(final_samples_planar, samples, audio_frame->nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt));
1270 
1271  // Fill input frame with sample data
1272  avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt, (uint8_t *) final_samples_planar,
1273  audio_encoder_buffer_size, 0);
1274 
1275  // Create output frame (and allocate arrays)
1276  frame_final->nb_samples = audio_input_frame_size;
1277  av_samples_alloc(frame_final->data, frame_final->linesize, info.channels, frame_final->nb_samples, audio_codec->sample_fmt, 0);
1278 
1279  // Convert audio samples
1280  int nb_samples = avresample_convert(avr_planar, // audio resample context
1281  frame_final->data, // output data pointers
1282  frame_final->linesize[0], // output plane size, in bytes. (0 if unknown)
1283  frame_final->nb_samples, // maximum number of samples that the output buffer can hold
1284  audio_frame->data, // input data pointers
1285  audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1286  audio_frame->nb_samples); // number of input samples to convert
1287 
1288  // Copy audio samples over original samples
1289  if (nb_samples > 0)
1290  memcpy(samples, frame_final->data[0], nb_samples * av_get_bytes_per_sample(audio_codec->sample_fmt) * info.channels);
1291 
1292  // deallocate AVFrame
1293  av_freep(&(audio_frame->data[0]));
1294  AV_FREE_FRAME(&audio_frame);
1295  all_queued_samples = NULL; // this array cleared with above call
1296 
1297  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)", "nb_samples", nb_samples, "", -1, "", -1, "", -1, "", -1, "", -1);
1298 
1299  } else {
1300  // Create a new array
1301  final_samples = new int16_t[audio_input_position * (av_get_bytes_per_sample(audio_codec->sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))];
1302 
1303  // Copy audio into buffer for frame
1304  memcpy(final_samples, samples, audio_input_position * av_get_bytes_per_sample(audio_codec->sample_fmt));
1305 
1306  // Init the nb_samples property
1307  frame_final->nb_samples = audio_input_frame_size;
1308 
1309  // Fill the final_frame AVFrame with audio (non planar)
1310  avcodec_fill_audio_frame(frame_final, audio_codec->channels, audio_codec->sample_fmt, (uint8_t *) final_samples,
1311  audio_encoder_buffer_size, 0);
1312  }
1313 
1314  // Increment PTS (in samples)
1315  write_audio_count += FFMIN(audio_input_frame_size, audio_input_position);
1316  frame_final->pts = write_audio_count; // Set the AVFrame's PTS
1317 
1318  // Init the packet
1319  AVPacket pkt;
1320  av_init_packet(&pkt);
1321  pkt.data = audio_encoder_buffer;
1322  pkt.size = audio_encoder_buffer_size;
1323 
1324  // Set the packet's PTS prior to encoding
1325  pkt.pts = pkt.dts = write_audio_count;
1326 
1327  /* encode the audio samples */
1328  int got_packet_ptr = 0;
1329  int error_code = avcodec_encode_audio2(audio_codec, &pkt, frame_final, &got_packet_ptr);
1330 
1331  /* if zero size, it means the image was buffered */
1332  if (error_code == 0 && got_packet_ptr) {
1333 
1334  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1335  // but it fixes lots of PTS related issues when I do this.
1336  pkt.pts = pkt.dts = write_audio_count;
1337 
1338  // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase)
1339  if (pkt.pts != AV_NOPTS_VALUE)
1340  pkt.pts = av_rescale_q(pkt.pts, audio_codec->time_base, audio_st->time_base);
1341  if (pkt.dts != AV_NOPTS_VALUE)
1342  pkt.dts = av_rescale_q(pkt.dts, audio_codec->time_base, audio_st->time_base);
1343  if (pkt.duration > 0)
1344  pkt.duration = av_rescale_q(pkt.duration, audio_codec->time_base, audio_st->time_base);
1345 
1346  // set stream
1347  pkt.stream_index = audio_st->index;
1348  pkt.flags |= AV_PKT_FLAG_KEY;
1349 
1350  /* write the compressed frame in the media file */
1351  int error_code = av_interleaved_write_frame(oc, &pkt);
1352  if (error_code < 0)
1353  {
1354  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1355  }
1356  }
1357 
1358  if (error_code < 0)
1359  {
1360  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1361  }
1362 
1363  // deallocate AVFrame
1364  av_freep(&(frame_final->data[0]));
1365  AV_FREE_FRAME(&frame_final);
1366 
1367  // deallocate memory for packet
1368  AV_FREE_PACKET(&pkt);
1369 
1370  // Reset position
1371  audio_input_position = 0;
1372  final = false;
1373  }
1374 
1375  // Delete arrays (if needed)
1376  if (all_resampled_samples) {
1377  av_freep(&all_resampled_samples);
1378  all_resampled_samples = NULL;
1379  }
1380  if (all_queued_samples) {
1381  av_freep(&all_queued_samples);
1382  all_queued_samples = NULL;
1383  }
1384 
1385  } // end task
1386 }
1387 
1388 // Allocate an AVFrame object
1389 AVFrame* FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer)
1390 {
1391  // Create an RGB AVFrame
1392  AVFrame *new_av_frame = NULL;
1393 
1394  // Allocate an AVFrame structure
1395  new_av_frame = AV_ALLOCATE_FRAME();
1396  if (new_av_frame == NULL)
1397  throw OutOfMemory("Could not allocate AVFrame", path);
1398 
1399  // Determine required buffer size and allocate buffer
1400  *buffer_size = avpicture_get_size(pix_fmt, width, height);
1401 
1402  // Create buffer (if not provided)
1403  if (!new_buffer)
1404  {
1405  // New Buffer
1406  new_buffer = (uint8_t*)av_malloc(*buffer_size * sizeof(uint8_t));
1407  // Attach buffer to AVFrame
1408  avpicture_fill((AVPicture *)new_av_frame, new_buffer, pix_fmt, width, height);
1409  new_av_frame->width = width;
1410  new_av_frame->height = height;
1411  new_av_frame->format = pix_fmt;
1412  }
1413 
1414  // return AVFrame
1415  return new_av_frame;
1416 }
1417 
1418 // process video frame
1419 void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame)
1420 {
1421  // Determine the height & width of the source image
1422  int source_image_width = frame->GetWidth();
1423  int source_image_height = frame->GetHeight();
1424 
1425  // Do nothing if size is 1x1 (i.e. no image in this frame)
1426  if (source_image_height == 1 && source_image_width == 1)
1427  return;
1428 
1429  // Init rescalers (if not initialized yet)
1430  if (image_rescalers.size() == 0)
1431  InitScalers(source_image_width, source_image_height);
1432 
1433  // Get a unique rescaler (for this thread)
1434  SwsContext *scaler = image_rescalers[rescaler_position];
1435  rescaler_position++;
1436  if (rescaler_position == num_of_rescalers)
1437  rescaler_position = 0;
1438 
1439  #pragma omp task firstprivate(frame, scaler, source_image_width, source_image_height)
1440  {
1441  // Allocate an RGB frame & final output frame
1442  int bytes_source = 0;
1443  int bytes_final = 0;
1444  AVFrame *frame_source = NULL;
1445  const uchar *pixels = NULL;
1446 
1447  // Get a list of pixels from source image
1448  pixels = frame->GetPixels();
1449 
1450  // Init AVFrame for source image & final (converted image)
1451  frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t*) pixels);
1452  AVFrame *frame_final = allocate_avframe(video_codec->pix_fmt, info.width, info.height, &bytes_final, NULL);
1453 
1454  // Fill with data
1455  avpicture_fill((AVPicture *) frame_source, (uint8_t*)pixels, PIX_FMT_RGBA, source_image_width, source_image_height);
1456  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::process_video_packet", "frame->number", frame->number, "bytes_source", bytes_source, "bytes_final", bytes_final, "", -1, "", -1, "", -1);
1457 
1458  // Resize & convert pixel format
1459  sws_scale(scaler, frame_source->data, frame_source->linesize, 0,
1460  source_image_height, frame_final->data, frame_final->linesize);
1461 
1462  // Add resized AVFrame to av_frames map
1463  #pragma omp critical (av_frames_section)
1464  add_avframe(frame, frame_final);
1465 
1466  // Deallocate memory
1467  AV_FREE_FRAME(&frame_source);
1468 
1469  } // end task
1470 
1471 }
1472 
1473 // write video frame
1474 bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame* frame_final)
1475 {
1476  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet", "frame->number", frame->number, "oc->oformat->flags & AVFMT_RAWPICTURE", oc->oformat->flags & AVFMT_RAWPICTURE, "", -1, "", -1, "", -1, "", -1);
1477 
1478  if (oc->oformat->flags & AVFMT_RAWPICTURE) {
1479  // Raw video case.
1480  AVPacket pkt;
1481  av_init_packet(&pkt);
1482 
1483  pkt.flags |= AV_PKT_FLAG_KEY;
1484  pkt.stream_index= video_st->index;
1485  pkt.data= (uint8_t*)frame_final->data;
1486  pkt.size= sizeof(AVPicture);
1487 
1488  // Increment PTS (in frames and scaled to the codec's timebase)
1489  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1490  pkt.pts = write_video_count;
1491 
1492  /* write the compressed frame in the media file */
1493  int error_code = av_interleaved_write_frame(oc, &pkt);
1494  if (error_code < 0)
1495  {
1496  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1497  return false;
1498  }
1499 
1500  // Deallocate packet
1501  AV_FREE_PACKET(&pkt);
1502 
1503  } else {
1504 
1505  AVPacket pkt;
1506  av_init_packet(&pkt);
1507  pkt.data = NULL;
1508  pkt.size = 0;
1509  pkt.pts = pkt.dts = AV_NOPTS_VALUE;
1510 
1511  // Pointer for video buffer (if using old FFmpeg version)
1512  uint8_t *video_outbuf = NULL;
1513 
1514  // Increment PTS (in frames and scaled to the codec's timebase)
1515  write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base);
1516 
1517  // Assign the initial AVFrame PTS from the frame counter
1518  frame_final->pts = write_video_count;
1519 
1520  /* encode the image */
1521  int got_packet_ptr = 0;
1522  int error_code = 0;
1523  #if LIBAVFORMAT_VERSION_MAJOR >= 54
1524  // Newer versions of FFMpeg
1525  error_code = avcodec_encode_video2(video_codec, &pkt, frame_final, &got_packet_ptr);
1526 
1527  #else
1528  // Older versions of FFmpeg (much sloppier)
1529 
1530  // Encode Picture and Write Frame
1531  int video_outbuf_size = 200000;
1532  video_outbuf = (uint8_t*) av_malloc(200000);
1533 
1534  /* encode the image */
1535  int out_size = avcodec_encode_video(video_codec, video_outbuf, video_outbuf_size, frame_final);
1536 
1537  /* if zero size, it means the image was buffered */
1538  if (out_size > 0) {
1539  if(video_codec->coded_frame->key_frame)
1540  pkt.flags |= AV_PKT_FLAG_KEY;
1541  pkt.data= video_outbuf;
1542  pkt.size= out_size;
1543 
1544  // got data back (so encode this frame)
1545  got_packet_ptr = 1;
1546  }
1547  #endif
1548 
1549  /* if zero size, it means the image was buffered */
1550  if (error_code == 0 && got_packet_ptr) {
1551 
1552  // Since the PTS can change during encoding, set the value again. This seems like a huge hack,
1553  // but it fixes lots of PTS related issues when I do this.
1554  //pkt.pts = pkt.dts = write_video_count;
1555 
1556  // set the timestamp
1557  if (pkt.pts != AV_NOPTS_VALUE)
1558  pkt.pts = av_rescale_q(pkt.pts, video_codec->time_base, video_st->time_base);
1559  if (pkt.dts != AV_NOPTS_VALUE)
1560  pkt.dts = av_rescale_q(pkt.dts, video_codec->time_base, video_st->time_base);
1561  if (pkt.duration > 0)
1562  pkt.duration = av_rescale_q(pkt.duration, video_codec->time_base, video_st->time_base);
1563  pkt.stream_index = video_st->index;
1564 
1565  /* write the compressed frame in the media file */
1566  int error_code = av_interleaved_write_frame(oc, &pkt);
1567  if (error_code < 0)
1568  {
1569  ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1);
1570  return false;
1571  }
1572  }
1573 
1574  // Deallocate memory (if needed)
1575  if (video_outbuf)
1576  delete[] video_outbuf;
1577 
1578  // Deallocate packet
1579  AV_FREE_PACKET(&pkt);
1580  }
1581 
1582  // Success
1583  return true;
1584 }
1585 
1586 // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
1588 {
1589  // output debug info
1590  av_dump_format(oc, 0, path.c_str(), 1);
1591 }
1592 
1593 // Init a collection of software rescalers (thread safe)
1594 void FFmpegWriter::InitScalers(int source_width, int source_height)
1595 {
1596  // Get the codec
1597  AVCodecContext *c;
1598  c = video_st->codec;
1599 
1600  // Init software rescalers vector (many of them, one for each thread)
1601  for (int x = 0; x < num_of_rescalers; x++)
1602  {
1603  // Init the software scaler from FFMpeg
1604  img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, c->pix_fmt, SWS_BILINEAR, NULL, NULL, NULL);
1605 
1606  // Add rescaler to vector
1607  image_rescalers.push_back(img_convert_ctx);
1608  }
1609 }
1610 
1611 // Set audio resample options
1612 void FFmpegWriter::ResampleAudio(int sample_rate, int channels) {
1613  original_sample_rate = sample_rate;
1614  original_channels = channels;
1615 }
1616 
1617 // Remove & deallocate all software scalers
1619 {
1620  // Close all rescalers
1621  for (int x = 0; x < num_of_rescalers; x++)
1622  sws_freeContext(image_rescalers[x]);
1623 
1624  // Clear vector
1625  image_rescalers.clear();
1626 }
#define AV_RESET_FRAME(av_frame)
int channels
The number of audio channels used in the audio stream.
Definition: WriterBase.h:72
A video stream (used to determine which type of stream)
Definition: FFmpegWriter.h:64
#define AV_FREE_FRAME(av_frame)
void SetOption(StreamType stream, string name, string value)
Set custom options (some codecs accept additional params). This must be called after the PrepareStrea...
int num
Numerator for the fraction.
Definition: Fraction.h:44
WriterInfo info
Information about the current media file.
Definition: WriterBase.h:92
void OutputStreamInfo()
Output the ffmpeg info about this format, streams, and codecs (i.e. dump format)
An audio stream (used to determine which type of stream)
Definition: FFmpegWriter.h:65
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: WriterBase.h:60
Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square) ...
Definition: WriterBase.h:61
Exception when an invalid # of audio channels are detected.
Definition: Exceptions.h:112
#define PIX_FMT_RGB24
string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: WriterBase.h:69
string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: WriterBase.h:63
void Reduce()
Reduce this fraction (i.e. 640/480 = 4/3)
Definition: Fraction.cpp:71
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:95
int width
The width of the video (in pixels)
Definition: WriterBase.h:57
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: WriterBase.h:70
#define OPEN_MP_NUM_PROCESSORS
void WriteFrame(std::shared_ptr< Frame > frame)
Add a frame to the stack waiting to be encoded.
Exception when encoding audio packet.
Definition: Exceptions.h:101
Exception when invalid sample rate is detected during encoding.
Definition: Exceptions.h:172
Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: WriterBase.h:66
void Open()
Open writer.
void SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate)
Set video export options.
virtual std::shared_ptr< Frame > GetFrame(int64_t number)=0
Exception when no valid codec is found for a file.
Definition: Exceptions.h:122
Exception when memory could not be allocated.
Definition: Exceptions.h:224
Exception when invalid encoding options are used.
Definition: Exceptions.h:162
#define AV_FREE_PACKET(av_packet)
Exception when no streams are found in the file.
Definition: Exceptions.h:192
void RemoveScalers()
Remove & deallocate all software scalers.
#define AV_ALLOCATE_FRAME()
bool top_field_first
Which interlaced field should be displayed first.
Definition: WriterBase.h:68
Exception for files that can not be found or opened.
Definition: Exceptions.h:132
FFmpegWriter(string path)
Constructor for FFmpegWriter. Throws one of the following exceptions.
void AppendDebugMethod(string method_name, string arg1_name, float arg1_value, string arg2_name, float arg2_value, string arg3_name, float arg3_value, string arg4_name, float arg4_value, string arg5_name, float arg5_value, string arg6_name, float arg6_value)
Append debug information.
Definition: ZmqLogger.cpp:162
This class represents a fraction.
Definition: Fraction.h:42
void ResampleAudio(int sample_rate, int channels)
Set audio resample options.
#define PIX_FMT_YUV420P
Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3) ...
Definition: WriterBase.h:62
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
#define PixelFormat
void WriteTrailer()
Write the file trailer (after all frames are written). This is called automatically by the Close() me...
#define av_err2str(errnum)
void WriteHeader()
Write the file header (after the options are set). This method is called automatically by the Open() ...
void Close()
Close the writer.
bool interlaced_frame
Are the contents of this frame interlaced.
Definition: WriterBase.h:67
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: WriterBase.h:71
bool has_video
Determines if this file has a video stream.
Definition: WriterBase.h:51
Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: WriterBase.h:59
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method) ...
Definition: ZmqLogger.cpp:38
This namespace is the default namespace for all code in the openshot library.
bool has_audio
Determines if this file has an audio stream.
Definition: WriterBase.h:52
#define PIX_FMT_RGBA
void SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate)
Set audio export options.
Exception when a writer is closed, and a frame is requested.
Definition: Exceptions.h:264
ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: WriterBase.h:73
void PrepareStreams()
Prepare & initialize streams and open codecs. This method is called automatically by the Open() metho...
int height
The height of the video (in pixels)
Definition: WriterBase.h:56
#define PIX_FMT_NONE
int den
Denominator for the fraction.
Definition: Fraction.h:45
Exception when no valid format is found for a file.
Definition: Exceptions.h:142
#define AUDIO_PACKET_ENCODING_SIZE
StreamType
This enumeration designates the type of stream when encoding (video or audio)
Definition: FFmpegWriter.h:62