OpenShot Library | libopenshot  0.1.9
CacheDisk.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for CacheDisk class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2014 OpenShot Studios, LLC
9  * <http://www.openshotstudios.com/>. This file is part of
10  * OpenShot Library (libopenshot), an open-source project dedicated to
11  * delivering high quality video editing and animation solutions to the
12  * world. For more information visit <http://www.openshot.org/>.
13  *
14  * OpenShot Library (libopenshot) is free software: you can redistribute it
15  * and/or modify it under the terms of the GNU Lesser General Public License
16  * as published by the Free Software Foundation, either version 3 of the
17  * License, or (at your option) any later version.
18  *
19  * OpenShot Library (libopenshot) is distributed in the hope that it will be
20  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22  * GNU Lesser General Public License for more details.
23  *
24  * You should have received a copy of the GNU Lesser General Public License
25  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
26  */
27 
28 #include "../include/CacheDisk.h"
29 
30 using namespace std;
31 using namespace openshot;
32 
33 // Default constructor, no max bytes
34 CacheDisk::CacheDisk(string cache_path, string format, float quality, float scale) : CacheBase(0) {
35  // Set cache type name
36  cache_type = "CacheDisk";
37  range_version = 0;
38  needs_range_processing = false;
39  frame_size_bytes = 0;
40  image_format = format;
41  image_quality = quality;
42  image_scale = scale;
43  max_bytes = 0;
44 
45  // Init path directory
46  InitPath(cache_path);
47 };
48 
49 // Constructor that sets the max bytes to cache
50 CacheDisk::CacheDisk(string cache_path, string format, float quality, float scale, int64_t max_bytes) : CacheBase(max_bytes) {
51  // Set cache type name
52  cache_type = "CacheDisk";
53  range_version = 0;
54  needs_range_processing = false;
55  frame_size_bytes = 0;
56  image_format = format;
57  image_quality = quality;
58  image_scale = scale;
59 
60  // Init path directory
61  InitPath(cache_path);
62 };
63 
64 // Initialize cache directory
65 void CacheDisk::InitPath(string cache_path) {
66  QString qpath;
67 
68  if (!cache_path.empty()) {
69  // Init QDir with cache directory
70  qpath = QString(cache_path.c_str());
71 
72  } else {
73  // Init QDir with user's temp directory
74  qpath = QDir::tempPath() + QString("/preview-cache/");
75  }
76 
77  // Init QDir with cache directory
78  path = QDir(qpath);
79 
80  // Check if cache directory exists
81  if (!path.exists())
82  // Create
83  path.mkpath(qpath);
84 }
85 
86 // Calculate ranges of frames
87 void CacheDisk::CalculateRanges() {
88  // Only calculate when something has changed
89  if (needs_range_processing) {
90 
91  // Create a scoped lock, to protect the cache from multiple threads
92  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
93 
94  // Sort ordered frame #s, and calculate JSON ranges
95  std::sort(ordered_frame_numbers.begin(), ordered_frame_numbers.end());
96 
97  // Clear existing JSON variable
98  Json::Value ranges = Json::Value(Json::arrayValue);
99 
100  // Increment range version
101  range_version++;
102 
103  vector<int64_t>::iterator itr_ordered;
104  int64_t starting_frame = *ordered_frame_numbers.begin();
105  int64_t ending_frame = *ordered_frame_numbers.begin();
106 
107  // Loop through all known frames (in sequential order)
108  for (itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end(); ++itr_ordered) {
109  int64_t frame_number = *itr_ordered;
110  if (frame_number - ending_frame > 1) {
111  // End of range detected
112  Json::Value range;
113 
114  // Add JSON object with start/end attributes
115  // Use strings, since int64_ts are supported in JSON
116  stringstream start_str;
117  start_str << starting_frame;
118  stringstream end_str;
119  end_str << ending_frame;
120  range["start"] = start_str.str();
121  range["end"] = end_str.str();
122  ranges.append(range);
123 
124  // Set new starting range
125  starting_frame = frame_number;
126  }
127 
128  // Set current frame as end of range, and keep looping
129  ending_frame = frame_number;
130  }
131 
132  // APPEND FINAL VALUE
133  Json::Value range;
134 
135  // Add JSON object with start/end attributes
136  // Use strings, since int64_ts are supported in JSON
137  stringstream start_str;
138  start_str << starting_frame;
139  stringstream end_str;
140  end_str << ending_frame;
141  range["start"] = start_str.str();
142  range["end"] = end_str.str();
143  ranges.append(range);
144 
145  // Cache range JSON as string
146  json_ranges = ranges.toStyledString();
147 
148  // Reset needs_range_processing
149  needs_range_processing = false;
150  }
151 }
152 
153 // Default destructor
155 {
156  frames.clear();
157  frame_numbers.clear();
158  ordered_frame_numbers.clear();
159 
160  // remove critical section
161  delete cacheCriticalSection;
162  cacheCriticalSection = NULL;
163 }
164 
165 // Add a Frame to the cache
166 void CacheDisk::Add(std::shared_ptr<Frame> frame)
167 {
168  // Create a scoped lock, to protect the cache from multiple threads
169  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
170  int64_t frame_number = frame->number;
171 
172  // Freshen frame if it already exists
173  if (frames.count(frame_number))
174  // Move frame to front of queue
175  MoveToFront(frame_number);
176 
177  else
178  {
179  // Add frame to queue and map
180  frames[frame_number] = frame_number;
181  frame_numbers.push_front(frame_number);
182  ordered_frame_numbers.push_back(frame_number);
183  needs_range_processing = true;
184 
185  // Save image to disk (if needed)
186  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
187  frame->Save(frame_path.toStdString(), image_scale, image_format, image_quality);
188  if (frame_size_bytes == 0) {
189  // Get compressed size of frame image (to correctly apply max size against)
190  QFile image_file(frame_path);
191  frame_size_bytes = image_file.size();
192  }
193 
194  // Save audio data (if needed)
195  if (frame->has_audio_data) {
196  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
197  QFile audio_file(audio_path);
198 
199  if (audio_file.open(QIODevice::WriteOnly)) {
200  QTextStream audio_stream(&audio_file);
201  audio_stream << frame->SampleRate() << endl;
202  audio_stream << frame->GetAudioChannelsCount() << endl;
203  audio_stream << frame->GetAudioSamplesCount() << endl;
204  audio_stream << frame->ChannelsLayout() << endl;
205 
206  // Loop through all samples
207  for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++)
208  {
209  // Get audio for this channel
210  float *samples = frame->GetAudioSamples(channel);
211  for (int sample = 0; sample < frame->GetAudioSamplesCount(); sample++)
212  audio_stream << samples[sample] << endl;
213  }
214 
215  }
216 
217  }
218 
219  // Clean up old frames
220  CleanUp();
221  }
222 }
223 
224 // Get a frame from the cache (or NULL shared_ptr if no frame is found)
225 std::shared_ptr<Frame> CacheDisk::GetFrame(int64_t frame_number)
226 {
227  // Create a scoped lock, to protect the cache from multiple threads
228  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
229 
230  // Does frame exists in cache?
231  if (frames.count(frame_number)) {
232  // Does frame exist on disk
233  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
234  if (path.exists(frame_path)) {
235 
236  // Load image file
237  std::shared_ptr<QImage> image = std::shared_ptr<QImage>(new QImage());
238  bool success = image->load(QString::fromStdString(frame_path.toStdString()));
239 
240  // Set pixel formatimage->
241  image = std::shared_ptr<QImage>(new QImage(image->convertToFormat(QImage::Format_RGBA8888)));
242 
243  // Create frame object
244  std::shared_ptr<Frame> frame(new Frame());
245  frame->number = frame_number;
246  frame->AddImage(image);
247 
248  // Get audio data (if found)
249  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
250  QFile audio_file(audio_path);
251  if (audio_file.exists()) {
252  // Open audio file
253  QTextStream in(&audio_file);
254  if (audio_file.open(QIODevice::ReadOnly)) {
255  int sample_rate = in.readLine().toInt();
256  int channels = in.readLine().toInt();
257  int sample_count = in.readLine().toInt();
258  int channel_layout = in.readLine().toInt();
259 
260  // Set basic audio properties
261  frame->ResizeAudio(channels, sample_count, sample_rate, (ChannelLayout) channel_layout);
262 
263  // Loop through audio samples and add to frame
264  int current_channel = 0;
265  int current_sample = 0;
266  float *channel_samples = new float[sample_count];
267  while (!in.atEnd()) {
268  // Add sample to channel array
269  channel_samples[current_sample] = in.readLine().toFloat();
270  current_sample++;
271 
272  if (current_sample == sample_count) {
273  // Add audio to frame
274  frame->AddAudio(true, current_channel, 0, channel_samples, sample_count, 1.0);
275 
276  // Increment channel, and reset sample position
277  current_channel++;
278  current_sample = 0;
279  }
280 
281  }
282  }
283  }
284 
285  // return the Frame object
286  return frame;
287  }
288  }
289 
290  // no Frame found
291  return std::shared_ptr<Frame>();
292 }
293 
294 // Get the smallest frame number (or NULL shared_ptr if no frame is found)
295 std::shared_ptr<Frame> CacheDisk::GetSmallestFrame()
296 {
297  // Create a scoped lock, to protect the cache from multiple threads
298  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
299  std::shared_ptr<openshot::Frame> f;
300 
301  // Loop through frame numbers
302  deque<int64_t>::iterator itr;
303  int64_t smallest_frame = -1;
304  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
305  {
306  if (*itr < smallest_frame || smallest_frame == -1)
307  smallest_frame = *itr;
308  }
309 
310  // Return frame
311  f = GetFrame(smallest_frame);
312 
313  return f;
314 }
315 
316 // Gets the maximum bytes value
318 {
319  // Create a scoped lock, to protect the cache from multiple threads
320  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
321 
322  int64_t total_bytes = 0;
323 
324  // Loop through frames, and calculate total bytes
325  deque<int64_t>::reverse_iterator itr;
326  for(itr = frame_numbers.rbegin(); itr != frame_numbers.rend(); ++itr)
327  total_bytes += frame_size_bytes;
328 
329  return total_bytes;
330 }
331 
332 // Remove a specific frame
333 void CacheDisk::Remove(int64_t frame_number)
334 {
335  Remove(frame_number, frame_number);
336 }
337 
338 // Remove range of frames
339 void CacheDisk::Remove(int64_t start_frame_number, int64_t end_frame_number)
340 {
341  // Create a scoped lock, to protect the cache from multiple threads
342  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
343 
344  // Loop through frame numbers
345  deque<int64_t>::iterator itr;
346  for(itr = frame_numbers.begin(); itr != frame_numbers.end();)
347  {
348  //deque<int64_t>::iterator current = itr++;
349  if (*itr >= start_frame_number && *itr <= end_frame_number)
350  {
351  // erase frame number
352  itr = frame_numbers.erase(itr);
353  } else
354  itr++;
355  }
356 
357  // Loop through ordered frame numbers
358  vector<int64_t>::iterator itr_ordered;
359  for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end();)
360  {
361  if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number)
362  {
363  // erase frame number
364  frames.erase(*itr_ordered);
365 
366  // Remove the image file (if it exists)
367  QString frame_path(path.path() + "/" + QString("%1.").arg(*itr_ordered) + QString(image_format.c_str()).toLower());
368  QFile image_file(frame_path);
369  if (image_file.exists())
370  image_file.remove();
371 
372  // Remove audio file (if it exists)
373  QString audio_path(path.path() + "/" + QString("%1").arg(*itr_ordered) + ".audio");
374  QFile audio_file(audio_path);
375  if (audio_file.exists())
376  audio_file.remove();
377 
378  itr_ordered = ordered_frame_numbers.erase(itr_ordered);
379  } else
380  itr_ordered++;
381  }
382 
383  // Needs range processing (since cache has changed)
384  needs_range_processing = true;
385 }
386 
387 // Move frame to front of queue (so it lasts longer)
388 void CacheDisk::MoveToFront(int64_t frame_number)
389 {
390  // Does frame exists in cache?
391  if (frames.count(frame_number))
392  {
393  // Create a scoped lock, to protect the cache from multiple threads
394  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
395 
396  // Loop through frame numbers
397  deque<int64_t>::iterator itr;
398  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
399  {
400  if (*itr == frame_number)
401  {
402  // erase frame number
403  frame_numbers.erase(itr);
404 
405  // add frame number to 'front' of queue
406  frame_numbers.push_front(frame_number);
407  break;
408  }
409  }
410  }
411 }
412 
413 // Clear the cache of all frames
415 {
416  // Create a scoped lock, to protect the cache from multiple threads
417  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
418 
419  // Clear all containers
420  frames.clear();
421  frame_numbers.clear();
422  ordered_frame_numbers.clear();
423  needs_range_processing = true;
424  frame_size_bytes = 0;
425 
426  // Delete cache directory, and recreate it
427  QString current_path = path.path();
428  path.removeRecursively();
429 
430  // Re-init folder
431  InitPath(current_path.toStdString());
432 }
433 
434 // Count the frames in the queue
436 {
437  // Create a scoped lock, to protect the cache from multiple threads
438  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
439 
440  // Return the number of frames in the cache
441  return frames.size();
442 }
443 
444 // Clean up cached frames that exceed the number in our max_bytes variable
445 void CacheDisk::CleanUp()
446 {
447  // Do we auto clean up?
448  if (max_bytes > 0)
449  {
450  // Create a scoped lock, to protect the cache from multiple threads
451  const GenericScopedLock<CriticalSection> lock(*cacheCriticalSection);
452 
453  while (GetBytes() > max_bytes && frame_numbers.size() > 20)
454  {
455  // Get the oldest frame number.
456  int64_t frame_to_remove = frame_numbers.back();
457 
458  // Remove frame_number and frame
459  Remove(frame_to_remove);
460  }
461  }
462 }
463 
464 // Generate JSON string of this object
465 string CacheDisk::Json() {
466 
467  // Return formatted string
468  return JsonValue().toStyledString();
469 }
470 
471 // Generate Json::JsonValue for this object
472 Json::Value CacheDisk::JsonValue() {
473 
474  // Proccess range data (if anything has changed)
475  CalculateRanges();
476 
477  // Create root json object
478  Json::Value root = CacheBase::JsonValue(); // get parent properties
479  root["type"] = cache_type;
480  root["path"] = path.path().toStdString();
481 
482  Json::Value version;
483  stringstream range_version_str;
484  range_version_str << range_version;
485  root["version"] = range_version_str.str();
486 
487  // Parse and append range data (if any)
488  Json::Value ranges;
489  Json::Reader reader;
490  bool success = reader.parse( json_ranges, ranges );
491  if (success)
492  root["ranges"] = ranges;
493 
494  // return JsonValue
495  return root;
496 }
497 
498 // Load JSON string into this object
499 void CacheDisk::SetJson(string value) {
500 
501  // Parse JSON string into JSON objects
502  Json::Value root;
503  Json::Reader reader;
504  bool success = reader.parse( value, root );
505  if (!success)
506  // Raise exception
507  throw InvalidJSON("JSON could not be parsed (or is invalid)", "");
508 
509  try
510  {
511  // Set all values that match
512  SetJsonValue(root);
513  }
514  catch (exception e)
515  {
516  // Error parsing JSON (or missing keys)
517  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", "");
518  }
519 }
520 
521 // Load Json::JsonValue into this object
522 void CacheDisk::SetJsonValue(Json::Value root) {
523 
524  // Close timeline before we do anything (this also removes all open and closing clips)
525  Clear();
526 
527  // Set parent data
529 
530  if (!root["type"].isNull())
531  cache_type = root["type"].asString();
532  if (!root["path"].isNull())
533  // Update duration of timeline
534  InitPath(root["path"].asString());
535 }
CriticalSection * cacheCriticalSection
Section lock for multiple threads.
Definition: CacheBase.h:52
string cache_type
This is a friendly type name of the derived cache instance.
Definition: CacheBase.h:48
void Add(std::shared_ptr< Frame > frame)
Add a Frame to the cache.
Definition: CacheDisk.cpp:166
int64_t Count()
Count the frames in the queue.
Definition: CacheDisk.cpp:435
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:115
int64_t GetBytes()
Gets the maximum bytes value.
Definition: CacheDisk.cpp:317
void SetJsonValue(Json::Value root)
Load Json::JsonValue into this object.
Definition: CacheDisk.cpp:522
std::shared_ptr< Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheDisk.cpp:225
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheDisk.cpp:333
Json::Value JsonValue()
Generate Json::JsonValue for this object.
Definition: CacheDisk.cpp:472
void MoveToFront(int64_t frame_number)
Move frame to front of queue (so it lasts longer)
Definition: CacheDisk.cpp:388
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:45
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
virtual Json::Value JsonValue()=0
Generate Json::JsonValue for this object.
Definition: CacheBase.cpp:54
void Clear()
Clear the cache of all frames.
Definition: CacheDisk.cpp:414
This namespace is the default namespace for all code in the openshot library.
CacheDisk(string cache_path, string format, float quality, float scale)
Default constructor, no max bytes.
Definition: CacheDisk.cpp:34
Exception for invalid JSON.
Definition: Exceptions.h:152
std::shared_ptr< Frame > GetSmallestFrame()
Get the smallest frame number.
Definition: CacheDisk.cpp:295
void SetJson(string value)
Load JSON string into this object.
Definition: CacheDisk.cpp:499
int64_t max_bytes
This is the max number of bytes to cache (0 = no limit)
Definition: CacheBase.h:49
virtual void SetJsonValue(Json::Value root)=0
Load Json::JsonValue into this object.
Definition: CacheBase.cpp:67
string Json()
Get and Set JSON methods.
Definition: CacheDisk.cpp:465