28 #include "../include/Frame.h" 34 Frame::Frame() : number(1), pixel_ratio(1,1), channels(2), width(1), height(1),
35 channel_layout(
LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false)
38 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, 0));
46 : number(number), pixel_ratio(1,1), channels(2), width(width), height(height),
50 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, 0));
58 number(number), pixel_ratio(1,1), channels(channels), width(1), height(1),
62 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, samples));
70 : number(number), pixel_ratio(1,1), channels(channels), width(width), height(height),
74 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(channels, samples));
92 image = std::shared_ptr<QImage>(
new QImage(*(other.image)));
93 audio = std::shared_ptr<juce::AudioSampleBuffer>(
new juce::AudioSampleBuffer(*(other.audio)));
94 pixel_ratio =
Fraction(other.pixel_ratio.
num, other.pixel_ratio.
den);
95 channels = other.channels;
96 channel_layout = other.channel_layout;
99 sample_rate = other.sample_rate;
102 if (other.wave_image)
103 wave_image = std::shared_ptr<QImage>(
new QImage(*(other.wave_image)));
116 if (!QApplication::instance()) {
119 static char* argv[1] = {NULL};
120 previewApp = std::shared_ptr<QApplication>(
new QApplication(argc, argv));
124 std::shared_ptr<QImage> previewImage =
GetImage();
127 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
130 int new_width = previewImage->size().width();
134 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
138 QWidget previewWindow;
139 previewWindow.setStyleSheet(
"background-color: #000000;");
144 previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
145 previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
146 layout.addWidget(&previewLabel);
149 previewWindow.setLayout(&layout);
150 previewWindow.show();
155 std::shared_ptr<QImage>
Frame::GetWaveform(
int width,
int height,
int Red,
int Green,
int Blue,
int Alpha)
161 QVector<QPointF> lines;
162 QVector<QPointF> labels;
165 int total_samples = audio->getNumSamples();
166 if (total_samples > 0)
169 int new_height = 200 * audio->getNumChannels();
170 int height_padding = 20 * (audio->getNumChannels() - 1);
171 int total_height = new_height + height_padding;
176 for (
int channel = 0; channel < audio->getNumChannels(); channel++)
181 const float *samples = audio->getReadPointer(channel);
183 for (
int sample = 0; sample < audio->getNumSamples(); sample++, X++)
186 float value = samples[sample] * 100;
191 lines.push_back(QPointF(X,Y));
192 lines.push_back(QPointF(X,Y-value));
196 lines.push_back(QPointF(X,Y));
197 lines.push_back(QPointF(X,Y));
202 labels.push_back(QPointF(5, Y - 5));
205 Y += (200 + height_padding);
210 wave_image = std::shared_ptr<QImage>(
new QImage(total_width, total_height, QImage::Format_RGBA8888));
211 wave_image->fill(QColor(0,0,0,0));
214 QPainter painter(wave_image.get());
217 painter.setPen(QColor(Red, Green, Blue, Alpha));
220 painter.drawLines(lines);
233 if (width != total_width || height != total_height) {
234 QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::FastTransformation);
235 wave_image = std::shared_ptr<QImage>(
new QImage(scaled_wave_image));
241 wave_image = std::shared_ptr<QImage>(
new QImage(width, height, QImage::Format_RGBA8888));
242 wave_image->fill(QColor(QString::fromStdString(
"#000000")));
260 wave_image =
GetWaveform(width, height, Red, Green, Blue, Alpha);
263 return wave_image->bits();
272 if (!QApplication::instance()) {
275 static char* argv[1] = {NULL};
276 previewApp = std::shared_ptr<QApplication>(
new QApplication(argc, argv));
280 QWidget previewWindow;
281 previewWindow.setStyleSheet(
"background-color: #000000;");
286 previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
287 previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
288 layout.addWidget(&previewLabel);
291 previewWindow.setLayout(&layout);
292 previewWindow.show();
304 return audio->getMagnitude(channel, sample, magnitude_range);
308 return audio->getMagnitude(sample, magnitude_range);
316 return audio->getWritePointer(channel);
322 float *output = NULL;
323 AudioSampleBuffer *buffer(audio.get());
324 int num_of_channels = audio->getNumChannels();
325 int num_of_samples = audio->getNumSamples();
328 if (new_sample_rate != sample_rate)
331 resampler->
SetBuffer(audio.get(), sample_rate, new_sample_rate);
337 num_of_samples = buffer->getNumSamples();
341 output =
new float[num_of_channels * num_of_samples];
345 for (
int channel = 0; channel < num_of_channels; channel++)
347 for (
int sample = 0; sample < num_of_samples; sample++)
350 output[position] = buffer->getReadPointer(channel)[sample];
358 *sample_count = num_of_samples;
368 float *output = NULL;
369 AudioSampleBuffer *buffer(audio.get());
370 int num_of_channels = audio->getNumChannels();
371 int num_of_samples = audio->getNumSamples();
374 if (new_sample_rate != sample_rate && resampler)
377 resampler->
SetBuffer(audio.get(), sample_rate, new_sample_rate);
383 num_of_samples = buffer->getNumSamples();
387 output =
new float[num_of_channels * num_of_samples];
391 for (
int sample = 0; sample < num_of_samples; sample++)
393 for (
int channel = 0; channel < num_of_channels; channel++)
396 output[position] = buffer->getReadPointer(channel)[sample];
404 *sample_count = num_of_samples;
413 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
415 return audio->getNumChannels();
423 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
425 return audio->getNumSamples();
438 int64_t total_bytes = 0;
440 total_bytes += (width * height *
sizeof(char) * 4);
443 total_bytes += (sample_rate / 24.0) *
sizeof(float);
459 return image->bits();
466 return image->scanLine(row);
472 pixel_ratio.
num = num;
473 pixel_ratio.
den = den;
489 double previous_samples = (sample_rate * fps_rate) * (number - 1);
490 double previous_samples_remainder = fmod(previous_samples, (
double)channels);
491 previous_samples -= previous_samples_remainder;
494 double total_samples = (sample_rate * fps_rate) * number;
495 double total_samples_remainder = fmod(total_samples, (
double)channels);
496 total_samples -= total_samples_remainder;
500 int samples_per_frame = round(total_samples - previous_samples);
501 return samples_per_frame;
531 return channel_layout;
536 void Frame::Save(
string path,
float scale,
string format,
int quality)
539 std::shared_ptr<QImage> previewImage =
GetImage();
542 if (abs(scale) > 1.001 || abs(scale) < 0.999)
544 int new_width = width;
545 int new_height = height;
548 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
551 int new_width = previewImage->size().width();
555 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
559 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width * scale, new_height * scale, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
563 previewImage->save(QString::fromStdString(path), format.c_str(), quality);
567 void Frame::Thumbnail(
string path,
int new_width,
int new_height,
string mask_path,
string overlay_path,
568 string background_color,
bool ignore_aspect,
string format,
int quality) {
571 std::shared_ptr<QImage> thumbnail = std::shared_ptr<QImage>(
new QImage(new_width, new_height, QImage::Format_RGBA8888));
572 thumbnail->fill(QColor(QString::fromStdString(background_color)));
575 QTransform transform;
576 QPainter painter(thumbnail.get());
577 painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing,
true);
581 std::shared_ptr<QImage> previewImage =
GetImage();
584 if (pixel_ratio.
num != 1 || pixel_ratio.
den != 1)
587 int aspect_width = previewImage->size().width();
588 int aspect_height = previewImage->size().height() * pixel_ratio.
Reciprocal().
ToDouble();
591 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(aspect_width, aspect_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
597 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
600 previewImage = std::shared_ptr<QImage>(
new QImage(previewImage->scaled(new_width, new_height, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
603 int x = (new_width - previewImage->size().width()) / 2.0;
604 int y = (new_height - previewImage->size().height()) / 2.0;
605 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
606 painter.drawImage(x, y, *previewImage);
610 if (overlay_path !=
"") {
612 std::shared_ptr<QImage> overlay = std::shared_ptr<QImage>(
new QImage());
613 overlay->load(QString::fromStdString(overlay_path));
616 overlay = std::shared_ptr<QImage>(
new QImage(overlay->convertToFormat(QImage::Format_RGBA8888)));
619 overlay = std::shared_ptr<QImage>(
new QImage(overlay->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
622 painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
623 painter.drawImage(0, 0, *overlay);
628 if (mask_path !=
"") {
630 std::shared_ptr<QImage> mask = std::shared_ptr<QImage>(
new QImage());
631 mask->load(QString::fromStdString(mask_path));
634 mask = std::shared_ptr<QImage>(
new QImage(mask->convertToFormat(QImage::Format_RGBA8888)));
637 mask = std::shared_ptr<QImage>(
new QImage(mask->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
640 mask->invertPixels();
643 unsigned char *pixels = (
unsigned char *) thumbnail->bits();
644 unsigned char *mask_pixels = (
unsigned char *) mask->bits();
648 for (
int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
651 int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
652 int Frame_Alpha = pixels[byte_index + 3];
653 int Mask_Value = constrain(Frame_Alpha - gray_value);
656 pixels[byte_index + 3] = Mask_Value;
665 thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
669 int Frame::constrain(
int color_value)
674 else if (color_value > 255)
684 const GenericScopedLock<CriticalSection> lock(addingImageSection);
685 #pragma omp critical (AddImage) 687 image = std::shared_ptr<QImage>(
new QImage(new_width, new_height, QImage::Format_RGBA8888));
690 image->fill(QColor(QString::fromStdString(color)));
693 width = image->width();
694 height = image->height();
699 void Frame::AddImage(
int new_width,
int new_height,
int bytes_per_pixel, QImage::Format type,
const unsigned char *pixels_)
702 const GenericScopedLock<CriticalSection> lock(addingImageSection);
703 int buffer_size = new_width * new_height * bytes_per_pixel;
704 qbuffer =
new unsigned char[buffer_size]();
707 memcpy((
unsigned char*)qbuffer, pixels_, buffer_size);
710 #pragma omp critical (AddImage) 712 image = std::shared_ptr<QImage>(
new QImage(qbuffer, new_width, new_height, new_width * bytes_per_pixel, type, (QImageCleanupFunction) &
openshot::Frame::cleanUpBuffer, (
void*) qbuffer));
715 if (image->format() != QImage::Format_RGBA8888)
716 image->convertToFormat(QImage::Format_RGBA8888);
719 width = image->width();
720 height = image->height();
733 const GenericScopedLock<CriticalSection> lock(addingImageSection);
734 #pragma omp critical (AddImage) 739 if (image->format() != QImage::Format_RGBA8888)
740 image->convertToFormat(QImage::Format_RGBA8888);
743 width = image->width();
744 height = image->height();
765 #pragma omp critical (AddImage) 766 if (image == new_image || image->size() != image->size() || image->format() != image->format())
772 const GenericScopedLock<CriticalSection> lock(addingImageSection);
773 #pragma omp critical (AddImage) 775 const unsigned char *pixels = image->bits();
776 const unsigned char *new_pixels = new_image->bits();
782 for (
int row = start; row < image->height(); row += 2) {
783 memcpy((
unsigned char *) pixels, new_pixels + (row * image->bytesPerLine()), image->bytesPerLine());
784 new_pixels += image->bytesPerLine();
788 width = image->width();
789 height = image->height();
799 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
802 audio->setSize(channels, length,
true,
true,
false);
803 channel_layout = layout;
808 void Frame::AddAudio(
bool replaceSamples,
int destChannel,
int destStartSample,
const float* source,
int numSamples,
float gainToApplyToSource = 1.0f) {
809 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
810 #pragma omp critical (adding_audio) 813 int new_length = destStartSample + numSamples;
814 int new_channel_length = audio->getNumChannels();
815 if (destChannel >= new_channel_length)
816 new_channel_length = destChannel + 1;
817 if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
818 audio->setSize(new_channel_length, new_length,
true,
true,
false);
822 audio->clear(destChannel, destStartSample, numSamples);
825 audio->addFrom(destChannel, destStartSample, source, numSamples, gainToApplyToSource);
831 void Frame::ApplyGainRamp(
int destChannel,
int destStartSample,
int numSamples,
float initial_gain = 0.0f,
float final_gain = 1.0f)
833 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
836 audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
850 #ifdef USE_IMAGEMAGICK 852 std::shared_ptr<Magick::Image> Frame::GetMagickImage()
860 QRgb
const *tmpBits = (
const QRgb*)image->bits();
863 std::shared_ptr<Magick::Image> magick_image = std::shared_ptr<Magick::Image>(
new Magick::Image(image->width(), image->height(),
"RGBA", Magick::CharPixel, tmpBits));
866 magick_image->backgroundColor(Magick::Color(
"none"));
867 magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
868 magick_image->matte(
true);
874 #ifdef USE_IMAGEMAGICK 876 void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
879 const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
884 qbuffer =
new unsigned char[bufferSize]();
885 unsigned char *buffer = (
unsigned char*)qbuffer;
890 Magick::PixelPacket *pixels = new_image->getPixels(0,0, new_image->columns(), new_image->rows());
891 for (
int n = 0, i = 0; n < new_image->columns() * new_image->rows(); n += 1, i += 4) {
892 buffer[i+0] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].red);
893 buffer[i+1] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].green);
894 buffer[i+2] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].blue);
895 buffer[i+3] = 255 - MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].opacity);
900 image = std::shared_ptr<QImage>(
new QImage(qbuffer, width, height, width * BPP, QImage::Format_RGBA8888, (QImageCleanupFunction) &
cleanUpBuffer, (
void*) qbuffer));
903 width = image->width();
904 height = image->height();
913 if (!audio->getNumSamples())
916 AudioDeviceManager deviceManager;
917 deviceManager.initialise (0,
923 AudioSourcePlayer audioSourcePlayer;
924 deviceManager.addAudioCallback (&audioSourcePlayer);
926 ScopedPointer<AudioBufferSource> my_source;
930 TimeSliceThread my_thread(
"Audio buffer thread");
933 my_thread.startThread();
935 AudioTransportSource transport1;
936 transport1.setSource (my_source,
939 (
double) sample_rate,
940 audio->getNumChannels());
941 transport1.setPosition (0);
942 transport1.setGain(1.0);
946 MixerAudioSource mixer;
947 mixer.addInputSource(&transport1,
false);
948 audioSourcePlayer.setSource (&mixer);
953 while (transport1.isPlaying())
955 cout <<
"playing" << endl;
959 cout <<
"DONE!!!" << endl;
962 transport1.setSource (0);
963 audioSourcePlayer.setSource (0);
964 my_thread.stopThread(500);
965 deviceManager.removeAudioCallback (&audioSourcePlayer);
966 deviceManager.closeAudioDevice();
967 deviceManager.removeAllChangeListeners();
968 deviceManager.dispatchPendingMessages();
970 cout <<
"End of Play()" << endl;
981 unsigned char* ptr_to_qbuffer = (
unsigned char*) info;
982 delete[] ptr_to_qbuffer;
989 const GenericScopedLock<CriticalSection> lock(addingAudioSection);
992 audio->setSize(channels, numSamples,
false,
true,
false);
void Thumbnail(string path, int new_width, int new_height, string mask_path, string overlay_path, string background_color, bool ignore_aspect, string format="png", int quality=100)
void SetBuffer(AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
int GetWidth()
Get height of image.
int num
Numerator for the fraction.
int GetAudioSamplesCount()
Get number of audio samples.
void AddColor(int new_width, int new_height, string color)
Add (or replace) pixel data to the frame (based on a solid color)
float * GetInterleavedAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
This class represents a single frame of video (i.e. image & audio data)
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
juce::AudioSampleBuffer * GetAudioSampleBuffer()
const unsigned char * GetPixels()
Get pixel data (as packets)
void Play()
Play audio samples for this frame.
void Save(string path, float scale, string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Fraction Reciprocal()
Return the reciprocal as a Fraction.
int64_t number
This is the frame number (starting at 1)
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
void ClearWaveform()
Clear the waveform image (and deallocate it's memory)
float * GetAudioSamples(int channel)
Get an array of sample data.
void SetFrameNumber(int64_t number)
Set frame number.
void AddAudioSilence(int numSamples)
Add audio silence.
bool has_audio_data
This frame has been loaded with audio data.
This class represents a fraction.
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
void DisplayWaveform()
Display the wave form.
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
int GetAudioChannelsCount()
Get number of audio channels.
This namespace is the default namespace for all code in the openshot library.
ChannelLayout ChannelsLayout()
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
~Frame()
Assignment operator.
AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
float * GetPlanarAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
int den
Denominator for the fraction.
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
bool has_image_data
This frame has been loaded with pixel data.
void ResizeAudio(int channels, int length, int sample_rate, ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
int GetHeight()
Get height of image.
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
int SampleRate()
Get the original sample rate of this frame's audio data.
This class is used to resample audio data for many sequential frames.