OpenShot Library | libopenshot  0.1.9
Frame.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for Frame class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2014 OpenShot Studios, LLC
9  * <http://www.openshotstudios.com/>. This file is part of
10  * OpenShot Library (libopenshot), an open-source project dedicated to
11  * delivering high quality video editing and animation solutions to the
12  * world. For more information visit <http://www.openshot.org/>.
13  *
14  * OpenShot Library (libopenshot) is free software: you can redistribute it
15  * and/or modify it under the terms of the GNU Lesser General Public License
16  * as published by the Free Software Foundation, either version 3 of the
17  * License, or (at your option) any later version.
18  *
19  * OpenShot Library (libopenshot) is distributed in the hope that it will be
20  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22  * GNU Lesser General Public License for more details.
23  *
24  * You should have received a copy of the GNU Lesser General Public License
25  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
26  */
27 
28 #include "../include/Frame.h"
29 
30 using namespace std;
31 using namespace openshot;
32 
33 // Constructor - blank frame (300x200 blank image, 48kHz audio silence)
34 Frame::Frame() : number(1), pixel_ratio(1,1), channels(2), width(1), height(1),
35  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false)
36 {
37  // Init the image magic and audio buffer
38  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, 0));
39 
40  // initialize the audio samples to zero (silence)
41  audio->clear();
42 };
43 
44 // Constructor - image only (48kHz audio silence)
45 Frame::Frame(int64_t number, int width, int height, string color)
46  : number(number), pixel_ratio(1,1), channels(2), width(width), height(height),
47  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false)
48 {
49  // Init the image magic and audio buffer
50  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, 0));
51 
52  // initialize the audio samples to zero (silence)
53  audio->clear();
54 };
55 
56 // Constructor - audio only (300x200 blank image)
57 Frame::Frame(int64_t number, int samples, int channels) :
58  number(number), pixel_ratio(1,1), channels(channels), width(1), height(1),
59  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false)
60 {
61  // Init the image magic and audio buffer
62  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, samples));
63 
64  // initialize the audio samples to zero (silence)
65  audio->clear();
66 };
67 
68 // Constructor - image & audio
69 Frame::Frame(int64_t number, int width, int height, string color, int samples, int channels)
70  : number(number), pixel_ratio(1,1), channels(channels), width(width), height(height),
71  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false)
72 {
73  // Init the image magic and audio buffer
74  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, samples));
75 
76  // initialize the audio samples to zero (silence)
77  audio->clear();
78 };
79 
80 
81 // Copy constructor
82 Frame::Frame ( const Frame &other )
83 {
84  // copy pointers and data
85  DeepCopy(other);
86 }
87 
88 // Copy data and pointers from another Frame instance
89 void Frame::DeepCopy(const Frame& other)
90 {
91  number = other.number;
92  image = std::shared_ptr<QImage>(new QImage(*(other.image)));
93  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(*(other.audio)));
94  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
95  channels = other.channels;
96  channel_layout = other.channel_layout;
99  sample_rate = other.sample_rate;
100 
101 
102  if (other.wave_image)
103  wave_image = std::shared_ptr<QImage>(new QImage(*(other.wave_image)));
104 }
105 
106 // Descructor
108  // Clear all pointers
109  image.reset();
110  audio.reset();
111 }
112 
113 // Display the frame image to the screen (primarily used for debugging reasons)
115 {
116  if (!QApplication::instance()) {
117  // Only create the QApplication once
118  static int argc = 1;
119  static char* argv[1] = {NULL};
120  previewApp = std::shared_ptr<QApplication>(new QApplication(argc, argv));
121  }
122 
123  // Get preview image
124  std::shared_ptr<QImage> previewImage = GetImage();
125 
126  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
127  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
128  {
129  // Calculate correct DAR (display aspect ratio)
130  int new_width = previewImage->size().width();
131  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
132 
133  // Resize to fix DAR
134  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
135  }
136 
137  // Create window
138  QWidget previewWindow;
139  previewWindow.setStyleSheet("background-color: #000000;");
140  QHBoxLayout layout;
141 
142  // Create label with current frame's image
143  QLabel previewLabel;
144  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
145  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
146  layout.addWidget(&previewLabel);
147 
148  // Show the window
149  previewWindow.setLayout(&layout);
150  previewWindow.show();
151  previewApp->exec();
152 }
153 
154 // Get an audio waveform image
155 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
156 {
157  // Clear any existing waveform image
158  ClearWaveform();
159 
160  // Init a list of lines
161  QVector<QPointF> lines;
162  QVector<QPointF> labels;
163 
164  // Calculate width of an image based on the # of samples
165  int total_samples = audio->getNumSamples();
166  if (total_samples > 0)
167  {
168  // If samples are present...
169  int new_height = 200 * audio->getNumChannels();
170  int height_padding = 20 * (audio->getNumChannels() - 1);
171  int total_height = new_height + height_padding;
172  int total_width = 0;
173 
174  // Loop through each audio channel
175  int Y = 100;
176  for (int channel = 0; channel < audio->getNumChannels(); channel++)
177  {
178  int X = 0;
179 
180  // Get audio for this channel
181  const float *samples = audio->getReadPointer(channel);
182 
183  for (int sample = 0; sample < audio->getNumSamples(); sample++, X++)
184  {
185  // Sample value (scaled to -100 to 100)
186  float value = samples[sample] * 100;
187 
188  // Append a line segment for each sample
189  if (value != 0.0) {
190  // LINE
191  lines.push_back(QPointF(X,Y));
192  lines.push_back(QPointF(X,Y-value));
193  }
194  else {
195  // DOT
196  lines.push_back(QPointF(X,Y));
197  lines.push_back(QPointF(X,Y));
198  }
199  }
200 
201  // Add Channel Label Coordinate
202  labels.push_back(QPointF(5, Y - 5));
203 
204  // Increment Y
205  Y += (200 + height_padding);
206  total_width = X;
207  }
208 
209  // Create blank image
210  wave_image = std::shared_ptr<QImage>(new QImage(total_width, total_height, QImage::Format_RGBA8888));
211  wave_image->fill(QColor(0,0,0,0));
212 
213  // Load QPainter with wave_image device
214  QPainter painter(wave_image.get());
215 
216  // Set pen color
217  painter.setPen(QColor(Red, Green, Blue, Alpha));
218 
219  // Draw the waveform
220  painter.drawLines(lines);
221  painter.end();
222 
223  // Loop through the channels labels (and draw the text)
224  // TODO: Configure Fonts in Qt5 correctly, so the drawText method does not crash
225 // painter.setFont(QFont(QString("Arial"), 16, 1, false));
226 // for (int channel = 0; channel < labels.size(); channel++) {
227 // stringstream label;
228 // label << "Channel " << channel;
229 // painter.drawText(labels.at(channel), QString::fromStdString(label.str()));
230 // }
231 
232  // Resize Image (if requested)
233  if (width != total_width || height != total_height) {
234  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::FastTransformation);
235  wave_image = std::shared_ptr<QImage>(new QImage(scaled_wave_image));
236  }
237  }
238  else
239  {
240  // No audio samples present
241  wave_image = std::shared_ptr<QImage>(new QImage(width, height, QImage::Format_RGBA8888));
242  wave_image->fill(QColor(QString::fromStdString("#000000")));
243  }
244 
245  // Return new image
246  return wave_image;
247 }
248 
249 // Clear the waveform image (and deallocate it's memory)
251 {
252  if (wave_image)
253  wave_image.reset();
254 }
255 
256 // Get an audio waveform image pixels
257 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
258 {
259  // Get audio wave form image
260  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
261 
262  // Return array of pixel packets
263  return wave_image->bits();
264 }
265 
266 // Display the wave form
268 {
269  // Get audio wave form image
270  GetWaveform(720, 480, 0, 123, 255, 255);
271 
272  if (!QApplication::instance()) {
273  // Only create the QApplication once
274  static int argc = 1;
275  static char* argv[1] = {NULL};
276  previewApp = std::shared_ptr<QApplication>(new QApplication(argc, argv));
277  }
278 
279  // Create window
280  QWidget previewWindow;
281  previewWindow.setStyleSheet("background-color: #000000;");
282  QHBoxLayout layout;
283 
284  // Create label with current frame's waveform image
285  QLabel previewLabel;
286  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
287  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
288  layout.addWidget(&previewLabel);
289 
290  // Show the window
291  previewWindow.setLayout(&layout);
292  previewWindow.show();
293  previewApp->exec();
294 
295  // Deallocate waveform image
296  ClearWaveform();
297 }
298 
299 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
300 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
301 {
302  if (channel > 0) {
303  // return average magnitude for a specific channel/sample range
304  return audio->getMagnitude(channel, sample, magnitude_range);
305 
306  } else {
307  // Return average magnitude for all channels
308  return audio->getMagnitude(sample, magnitude_range);
309  }
310 }
311 
312 // Get an array of sample data
313 float* Frame::GetAudioSamples(int channel)
314 {
315  // return JUCE audio data for this channel
316  return audio->getWritePointer(channel);
317 }
318 
319 // Get a planar array of sample data, using any sample rate
320 float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
321 {
322  float *output = NULL;
323  AudioSampleBuffer *buffer(audio.get());
324  int num_of_channels = audio->getNumChannels();
325  int num_of_samples = audio->getNumSamples();
326 
327  // Resample to new sample rate (if needed)
328  if (new_sample_rate != sample_rate)
329  {
330  // YES, RESAMPLE AUDIO
331  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
332 
333  // Resample data, and return new buffer pointer
334  buffer = resampler->GetResampledBuffer();
335 
336  // Update num_of_samples
337  num_of_samples = buffer->getNumSamples();
338  }
339 
340  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
341  output = new float[num_of_channels * num_of_samples];
342  int position = 0;
343 
344  // Loop through samples in each channel (combining them)
345  for (int channel = 0; channel < num_of_channels; channel++)
346  {
347  for (int sample = 0; sample < num_of_samples; sample++)
348  {
349  // Add sample to output array
350  output[position] = buffer->getReadPointer(channel)[sample];
351 
352  // increment position
353  position++;
354  }
355  }
356 
357  // Update sample count (since it might have changed due to resampling)
358  *sample_count = num_of_samples;
359 
360  // return combined array
361  return output;
362 }
363 
364 
365 // Get an array of sample data (all channels interleaved together), using any sample rate
366 float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
367 {
368  float *output = NULL;
369  AudioSampleBuffer *buffer(audio.get());
370  int num_of_channels = audio->getNumChannels();
371  int num_of_samples = audio->getNumSamples();
372 
373  // Resample to new sample rate (if needed)
374  if (new_sample_rate != sample_rate && resampler)
375  {
376  // YES, RESAMPLE AUDIO
377  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
378 
379  // Resample data, and return new buffer pointer
380  buffer = resampler->GetResampledBuffer();
381 
382  // Update num_of_samples
383  num_of_samples = buffer->getNumSamples();
384  }
385 
386  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
387  output = new float[num_of_channels * num_of_samples];
388  int position = 0;
389 
390  // Loop through samples in each channel (combining them)
391  for (int sample = 0; sample < num_of_samples; sample++)
392  {
393  for (int channel = 0; channel < num_of_channels; channel++)
394  {
395  // Add sample to output array
396  output[position] = buffer->getReadPointer(channel)[sample];
397 
398  // increment position
399  position++;
400  }
401  }
402 
403  // Update sample count (since it might have changed due to resampling)
404  *sample_count = num_of_samples;
405 
406  // return combined array
407  return output;
408 }
409 
410 // Get number of audio channels
412 {
413  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
414  if (audio)
415  return audio->getNumChannels();
416  else
417  return 0;
418 }
419 
420 // Get number of audio samples
422 {
423  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
424  if (audio)
425  return audio->getNumSamples();
426  else
427  return 0;
428 }
429 
430 juce::AudioSampleBuffer *Frame::GetAudioSampleBuffer()
431 {
432  return audio.get();
433 }
434 
435 // Get the size in bytes of this frame (rough estimate)
437 {
438  int64_t total_bytes = 0;
439  if (image)
440  total_bytes += (width * height * sizeof(char) * 4);
441  if (audio) {
442  // approximate audio size (sample rate / 24 fps)
443  total_bytes += (sample_rate / 24.0) * sizeof(float);
444  }
445 
446  // return size of this frame
447  return total_bytes;
448 }
449 
450 // Get pixel data (as packets)
451 const unsigned char* Frame::GetPixels()
452 {
453  // Check for blank image
454  if (!image)
455  // Fill with black
456  AddColor(width, height, "#000000");
457 
458  // Return array of pixel packets
459  return image->bits();
460 }
461 
462 // Get pixel data (for only a single scan-line)
463 const unsigned char* Frame::GetPixels(int row)
464 {
465  // Return array of pixel packets
466  return image->scanLine(row);
467 }
468 
469 // Set Pixel Aspect Ratio
470 void Frame::SetPixelRatio(int num, int den)
471 {
472  pixel_ratio.num = num;
473  pixel_ratio.den = den;
474 }
475 
476 // Set frame number
477 void Frame::SetFrameNumber(int64_t new_number)
478 {
479  number = new_number;
480 }
481 
482 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
483 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
484 {
485  // Get the total # of samples for the previous frame, and the current frame (rounded)
486  double fps_rate = fps.Reciprocal().ToDouble();
487 
488  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
489  double previous_samples = (sample_rate * fps_rate) * (number - 1);
490  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
491  previous_samples -= previous_samples_remainder;
492 
493  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
494  double total_samples = (sample_rate * fps_rate) * number;
495  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
496  total_samples -= total_samples_remainder;
497 
498  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
499  // be evenly divided into frames, so each frame can have have different # of samples.
500  int samples_per_frame = round(total_samples - previous_samples);
501  return samples_per_frame;
502 }
503 
504 // Calculate the # of samples per video frame (for the current frame number)
505 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
506 {
507  return GetSamplesPerFrame(number, fps, sample_rate, channels);
508 }
509 
510 // Get height of image
512 {
513  return height;
514 }
515 
516 // Get height of image
518 {
519  return width;
520 }
521 
522 // Get the original sample rate of this frame's audio data
524 {
525  return sample_rate;
526 }
527 
528 // Get the original sample rate of this frame's audio data
530 {
531  return channel_layout;
532 }
533 
534 
535 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
536 void Frame::Save(string path, float scale, string format, int quality)
537 {
538  // Get preview image
539  std::shared_ptr<QImage> previewImage = GetImage();
540 
541  // scale image if needed
542  if (abs(scale) > 1.001 || abs(scale) < 0.999)
543  {
544  int new_width = width;
545  int new_height = height;
546 
547  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
548  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
549  {
550  // Calculate correct DAR (display aspect ratio)
551  int new_width = previewImage->size().width();
552  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
553 
554  // Resize to fix DAR
555  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
556  }
557 
558  // Resize image
559  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width * scale, new_height * scale, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
560  }
561 
562  // Save image
563  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
564 }
565 
566 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
567 void Frame::Thumbnail(string path, int new_width, int new_height, string mask_path, string overlay_path,
568  string background_color, bool ignore_aspect, string format, int quality) {
569 
570  // Create blank thumbnail image & fill background color
571  std::shared_ptr<QImage> thumbnail = std::shared_ptr<QImage>(new QImage(new_width, new_height, QImage::Format_RGBA8888));
572  thumbnail->fill(QColor(QString::fromStdString(background_color)));
573 
574  // Create transform and painter
575  QTransform transform;
576  QPainter painter(thumbnail.get());
577  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
578 
579 
580  // Get preview image
581  std::shared_ptr<QImage> previewImage = GetImage();
582 
583  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
584  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
585  {
586  // Calculate correct DAR (display aspect ratio)
587  int aspect_width = previewImage->size().width();
588  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
589 
590  // Resize to fix DAR
591  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(aspect_width, aspect_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
592  }
593 
594  // Resize frame image
595  if (ignore_aspect)
596  // Ignore aspect ratio
597  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
598  else
599  // Maintain aspect ratio
600  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
601 
602  // Composite frame image onto background (centered)
603  int x = (new_width - previewImage->size().width()) / 2.0; // center
604  int y = (new_height - previewImage->size().height()) / 2.0; // center
605  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
606  painter.drawImage(x, y, *previewImage);
607 
608 
609  // Overlay Image (if any)
610  if (overlay_path != "") {
611  // Open overlay
612  std::shared_ptr<QImage> overlay = std::shared_ptr<QImage>(new QImage());
613  overlay->load(QString::fromStdString(overlay_path));
614 
615  // Set pixel format
616  overlay = std::shared_ptr<QImage>(new QImage(overlay->convertToFormat(QImage::Format_RGBA8888)));
617 
618  // Resize to fit
619  overlay = std::shared_ptr<QImage>(new QImage(overlay->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
620 
621  // Composite onto thumbnail
622  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
623  painter.drawImage(0, 0, *overlay);
624  }
625 
626 
627  // Mask Image (if any)
628  if (mask_path != "") {
629  // Open mask
630  std::shared_ptr<QImage> mask = std::shared_ptr<QImage>(new QImage());
631  mask->load(QString::fromStdString(mask_path));
632 
633  // Set pixel format
634  mask = std::shared_ptr<QImage>(new QImage(mask->convertToFormat(QImage::Format_RGBA8888)));
635 
636  // Resize to fit
637  mask = std::shared_ptr<QImage>(new QImage(mask->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
638 
639  // Negate mask
640  mask->invertPixels();
641 
642  // Get pixels
643  unsigned char *pixels = (unsigned char *) thumbnail->bits();
644  unsigned char *mask_pixels = (unsigned char *) mask->bits();
645 
646  // Convert the mask image to grayscale
647  // Loop through pixels
648  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
649  {
650  // Get the RGB values from the pixel
651  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
652  int Frame_Alpha = pixels[byte_index + 3];
653  int Mask_Value = constrain(Frame_Alpha - gray_value);
654 
655  // Set all alpha pixels to gray value
656  pixels[byte_index + 3] = Mask_Value;
657  }
658  }
659 
660 
661  // End painter
662  painter.end();
663 
664  // Save image
665  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
666 }
667 
668 // Constrain a color value from 0 to 255
669 int Frame::constrain(int color_value)
670 {
671  // Constrain new color from 0 to 255
672  if (color_value < 0)
673  color_value = 0;
674  else if (color_value > 255)
675  color_value = 255;
676 
677  return color_value;
678 }
679 
680 // Add (or replace) pixel data to the frame (based on a solid color)
681 void Frame::AddColor(int new_width, int new_height, string color)
682 {
683  // Create new image object, and fill with pixel data
684  const GenericScopedLock<CriticalSection> lock(addingImageSection);
685  #pragma omp critical (AddImage)
686  {
687  image = std::shared_ptr<QImage>(new QImage(new_width, new_height, QImage::Format_RGBA8888));
688 
689  // Fill with solid color
690  image->fill(QColor(QString::fromStdString(color)));
691  }
692  // Update height and width
693  width = image->width();
694  height = image->height();
695  has_image_data = true;
696 }
697 
698 // Add (or replace) pixel data to the frame
699 void Frame::AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
700 {
701  // Create new buffer
702  const GenericScopedLock<CriticalSection> lock(addingImageSection);
703  int buffer_size = new_width * new_height * bytes_per_pixel;
704  qbuffer = new unsigned char[buffer_size]();
705 
706  // Copy buffer data
707  memcpy((unsigned char*)qbuffer, pixels_, buffer_size);
708 
709  // Create new image object, and fill with pixel data
710  #pragma omp critical (AddImage)
711  {
712  image = std::shared_ptr<QImage>(new QImage(qbuffer, new_width, new_height, new_width * bytes_per_pixel, type, (QImageCleanupFunction) &openshot::Frame::cleanUpBuffer, (void*) qbuffer));
713 
714  // Always convert to RGBA8888 (if different)
715  if (image->format() != QImage::Format_RGBA8888)
716  image->convertToFormat(QImage::Format_RGBA8888);
717 
718  // Update height and width
719  width = image->width();
720  height = image->height();
721  has_image_data = true;
722  }
723 }
724 
725 // Add (or replace) pixel data to the frame
726 void Frame::AddImage(std::shared_ptr<QImage> new_image)
727 {
728  // Ignore blank images
729  if (!new_image)
730  return;
731 
732  // assign image data
733  const GenericScopedLock<CriticalSection> lock(addingImageSection);
734  #pragma omp critical (AddImage)
735  {
736  image = new_image;
737 
738  // Always convert to RGBA8888 (if different)
739  if (image->format() != QImage::Format_RGBA8888)
740  image->convertToFormat(QImage::Format_RGBA8888);
741 
742  // Update height and width
743  width = image->width();
744  height = image->height();
745  has_image_data = true;
746  }
747 }
748 
749 // Add (or replace) pixel data to the frame (for only the odd or even lines)
750 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
751 {
752  // Ignore blank new_image
753  if (!new_image)
754  return;
755 
756  // Check for blank source image
757  if (!image) {
758  // Replace the blank source image
759  AddImage(new_image);
760 
761  } else {
762 
763  // Ignore image of different sizes or formats
764  bool ret=false;
765  #pragma omp critical (AddImage)
766  if (image == new_image || image->size() != image->size() || image->format() != image->format())
767  ret=true;
768  if (ret)
769  return;
770 
771  // Get the frame's image
772  const GenericScopedLock<CriticalSection> lock(addingImageSection);
773  #pragma omp critical (AddImage)
774  {
775  const unsigned char *pixels = image->bits();
776  const unsigned char *new_pixels = new_image->bits();
777 
778  // Loop through the scanlines of the image (even or odd)
779  int start = 0;
780  if (only_odd_lines)
781  start = 1;
782  for (int row = start; row < image->height(); row += 2) {
783  memcpy((unsigned char *) pixels, new_pixels + (row * image->bytesPerLine()), image->bytesPerLine());
784  new_pixels += image->bytesPerLine();
785  }
786 
787  // Update height and width
788  width = image->width();
789  height = image->height();
790  has_image_data = true;
791  }
792  }
793 }
794 
795 
796 // Resize audio container to hold more (or less) samples and channels
797 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
798 {
799  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
800 
801  // Resize JUCE audio buffer
802  audio->setSize(channels, length, true, true, false);
803  channel_layout = layout;
804  sample_rate = rate;
805 }
806 
807 // Add audio samples to a specific channel
808 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
809  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
810  #pragma omp critical (adding_audio)
811  {
812  // Extend audio container to hold more (or less) samples and channels.. if needed
813  int new_length = destStartSample + numSamples;
814  int new_channel_length = audio->getNumChannels();
815  if (destChannel >= new_channel_length)
816  new_channel_length = destChannel + 1;
817  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
818  audio->setSize(new_channel_length, new_length, true, true, false);
819 
820  // Clear the range of samples first (if needed)
821  if (replaceSamples)
822  audio->clear(destChannel, destStartSample, numSamples);
823 
824  // Add samples to frame's audio buffer
825  audio->addFrom(destChannel, destStartSample, source, numSamples, gainToApplyToSource);
826  has_audio_data = true;
827  }
828 }
829 
830 // Apply gain ramp (i.e. fading volume)
831 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
832 {
833  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
834 
835  // Apply gain ramp
836  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
837 }
838 
839 // Get pointer to Magick++ image object
840 std::shared_ptr<QImage> Frame::GetImage()
841 {
842  // Check for blank image
843  if (!image)
844  // Fill with black
845  AddColor(width, height, "#000000");
846 
847  return image;
848 }
849 
850 #ifdef USE_IMAGEMAGICK
851 // Get pointer to ImageMagick image object
852 std::shared_ptr<Magick::Image> Frame::GetMagickImage()
853 {
854  // Check for blank image
855  if (!image)
856  // Fill with black
857  AddColor(width, height, "#000000");
858 
859  // Get the pixels from the frame image
860  QRgb const *tmpBits = (const QRgb*)image->bits();
861 
862  // Create new image object, and fill with pixel data
863  std::shared_ptr<Magick::Image> magick_image = std::shared_ptr<Magick::Image>(new Magick::Image(image->width(), image->height(),"RGBA", Magick::CharPixel, tmpBits));
864 
865  // Give image a transparent background color
866  magick_image->backgroundColor(Magick::Color("none"));
867  magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
868  magick_image->matte(true);
869 
870  return magick_image;
871 }
872 #endif
873 
874 #ifdef USE_IMAGEMAGICK
875 // Get pointer to QImage of frame
876 void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
877 {
878  const int BPP = 4;
879  const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
880 
881  /// Use realloc for fast memory allocation.
882  /// TODO: consider locking the buffer for mt safety
883  //qbuffer = reinterpret_cast<unsigned char*>(realloc(qbuffer, bufferSize));
884  qbuffer = new unsigned char[bufferSize]();
885  unsigned char *buffer = (unsigned char*)qbuffer;
886 
887  // Iterate through the pixel packets, and load our own buffer
888  // Each color needs to be scaled to 8 bit (using the ImageMagick built-in ScaleQuantumToChar function)
889  int numcopied = 0;
890  Magick::PixelPacket *pixels = new_image->getPixels(0,0, new_image->columns(), new_image->rows());
891  for (int n = 0, i = 0; n < new_image->columns() * new_image->rows(); n += 1, i += 4) {
892  buffer[i+0] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].red);
893  buffer[i+1] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].green);
894  buffer[i+2] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].blue);
895  buffer[i+3] = 255 - MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].opacity);
896  numcopied+=4;
897  }
898 
899  // Create QImage of frame data
900  image = std::shared_ptr<QImage>(new QImage(qbuffer, width, height, width * BPP, QImage::Format_RGBA8888, (QImageCleanupFunction) &cleanUpBuffer, (void*) qbuffer));
901 
902  // Update height and width
903  width = image->width();
904  height = image->height();
905  has_image_data = true;
906 }
907 #endif
908 
909 // Play audio samples for this frame
911 {
912  // Check if samples are present
913  if (!audio->getNumSamples())
914  return;
915 
916  AudioDeviceManager deviceManager;
917  deviceManager.initialise (0, /* number of input channels */
918  2, /* number of output channels */
919  0, /* no XML settings.. */
920  true /* select default device on failure */);
921  //deviceManager.playTestSound();
922 
923  AudioSourcePlayer audioSourcePlayer;
924  deviceManager.addAudioCallback (&audioSourcePlayer);
925 
926  ScopedPointer<AudioBufferSource> my_source;
927  my_source = new AudioBufferSource(audio.get());
928 
929  // Create TimeSliceThread for audio buffering
930  TimeSliceThread my_thread("Audio buffer thread");
931 
932  // Start thread
933  my_thread.startThread();
934 
935  AudioTransportSource transport1;
936  transport1.setSource (my_source,
937  5000, // tells it to buffer this many samples ahead
938  &my_thread,
939  (double) sample_rate,
940  audio->getNumChannels()); // sample rate of source
941  transport1.setPosition (0);
942  transport1.setGain(1.0);
943 
944 
945  // Create MIXER
946  MixerAudioSource mixer;
947  mixer.addInputSource(&transport1, false);
948  audioSourcePlayer.setSource (&mixer);
949 
950  // Start transports
951  transport1.start();
952 
953  while (transport1.isPlaying())
954  {
955  cout << "playing" << endl;
956  usleep(1000000);
957  }
958 
959  cout << "DONE!!!" << endl;
960 
961  transport1.stop();
962  transport1.setSource (0);
963  audioSourcePlayer.setSource (0);
964  my_thread.stopThread(500);
965  deviceManager.removeAudioCallback (&audioSourcePlayer);
966  deviceManager.closeAudioDevice();
967  deviceManager.removeAllChangeListeners();
968  deviceManager.dispatchPendingMessages();
969 
970  cout << "End of Play()" << endl;
971 
972 
973 }
974 
975 // Clean up buffer after QImage is deleted
976 void Frame::cleanUpBuffer(void *info)
977 {
978  if (info)
979  {
980  // Remove buffer since QImage tells us to
981  unsigned char* ptr_to_qbuffer = (unsigned char*) info;
982  delete[] ptr_to_qbuffer;
983  }
984 }
985 
986 // Add audio silence
987 void Frame::AddAudioSilence(int numSamples)
988 {
989  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
990 
991  // Resize audio container
992  audio->setSize(channels, numSamples, false, true, false);
993  audio->clear();
994  has_audio_data = true;
995 }
void Thumbnail(string path, int new_width, int new_height, string mask_path, string overlay_path, string background_color, bool ignore_aspect, string format="png", int quality=100)
Definition: Frame.cpp:567
void SetBuffer(AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
int GetWidth()
Get height of image.
Definition: Frame.cpp:517
int num
Numerator for the fraction.
Definition: Fraction.h:44
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:421
void AddColor(int new_width, int new_height, string color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:681
float * GetInterleavedAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
Definition: Frame.cpp:366
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:115
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:257
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:430
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:451
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:910
void Save(string path, float scale, string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
Definition: Frame.cpp:536
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:89
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:808
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:114
Fraction Reciprocal()
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:81
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:136
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
void ClearWaveform()
Clear the waveform image (and deallocate it&#39;s memory)
Definition: Frame.cpp:250
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:313
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:477
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:987
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:137
This class represents a fraction.
Definition: Fraction.h:42
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:976
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:699
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
Definition: Frame.cpp:34
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:831
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:267
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:470
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:411
This namespace is the default namespace for all code in the openshot library.
ChannelLayout ChannelsLayout()
Definition: Frame.cpp:529
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:840
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:155
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:436
~Frame()
Assignment operator.
Definition: Frame.cpp:107
AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
float * GetPlanarAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:320
int den
Denominator for the fraction.
Definition: Fraction.h:45
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
Definition: Frame.cpp:300
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:505
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:138
void ResizeAudio(int channels, int length, int sample_rate, ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:797
int GetHeight()
Get height of image.
Definition: Frame.cpp:511
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int SampleRate()
Get the original sample rate of this frame&#39;s audio data.
Definition: Frame.cpp:523
This class is used to resample audio data for many sequential frames.