Package CedarBackup3 :: Package extend :: Module mbox
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup3.extend.mbox

   1  # -*- coding: iso-8859-1 -*- 
   2  # vim: set ft=python ts=3 sw=3 expandtab: 
   3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
   4  # 
   5  #              C E D A R 
   6  #          S O L U T I O N S       "Software done right." 
   7  #           S O F T W A R E 
   8  # 
   9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  10  # 
  11  # Copyright (c) 2006-2007,2010,2015 Kenneth J. Pronovici. 
  12  # All rights reserved. 
  13  # 
  14  # This program is free software; you can redistribute it and/or 
  15  # modify it under the terms of the GNU General Public License, 
  16  # Version 2, as published by the Free Software Foundation. 
  17  # 
  18  # This program is distributed in the hope that it will be useful, 
  19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
  20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
  21  # 
  22  # Copies of the GNU General Public License are available from 
  23  # the Free Software Foundation website, http://www.gnu.org/. 
  24  # 
  25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  26  # 
  27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
  28  # Language : Python 3 (>= 3.4) 
  29  # Project  : Official Cedar Backup Extensions 
  30  # Purpose  : Provides an extension to back up mbox email files. 
  31  # 
  32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  33   
  34  ######################################################################## 
  35  # Module documentation 
  36  ######################################################################## 
  37   
  38  """ 
  39  Provides an extension to back up mbox email files. 
  40   
  41  Backing up email 
  42  ================ 
  43   
  44     Email folders (often stored as mbox flatfiles) are not well-suited being backed 
  45     up with an incremental backup like the one offered by Cedar Backup.  This is 
  46     because mbox files often change on a daily basis, forcing the incremental 
  47     backup process to back them up every day in order to avoid losing data.  This 
  48     can result in quite a bit of wasted space when backing up large folders.  (Note 
  49     that the alternative maildir format does not share this problem, since it 
  50     typically uses one file per message.) 
  51   
  52     One solution to this problem is to design a smarter incremental backup process, 
  53     which backs up baseline content on the first day of the week, and then backs up 
  54     only new messages added to that folder on every other day of the week.  This way, 
  55     the backup for any single day is only as large as the messages placed into the 
  56     folder on that day.  The backup isn't as "perfect" as the incremental backup 
  57     process, because it doesn't preserve information about messages deleted from 
  58     the backed-up folder.  However, it should be much more space-efficient, and 
  59     in a recovery situation, it seems better to restore too much data rather 
  60     than too little. 
  61   
  62  What is this extension? 
  63  ======================= 
  64   
  65     This is a Cedar Backup extension used to back up mbox email files via the Cedar 
  66     Backup command line.  Individual mbox files or directories containing mbox 
  67     files can be backed up using the same collect modes allowed for filesystems in 
  68     the standard Cedar Backup collect action: weekly, daily, incremental.  It 
  69     implements the "smart" incremental backup process discussed above, using 
  70     functionality provided by the C{grepmail} utility. 
  71   
  72     This extension requires a new configuration section <mbox> and is intended to 
  73     be run either immediately before or immediately after the standard collect 
  74     action.  Aside from its own configuration, it requires the options and collect 
  75     configuration sections in the standard Cedar Backup configuration file. 
  76   
  77     The mbox action is conceptually similar to the standard collect action, 
  78     except that mbox directories are not collected recursively.  This implies 
  79     some configuration changes (i.e. there's no need for global exclusions or an 
  80     ignore file).  If you back up a directory, all of the mbox files in that 
  81     directory are backed up into a single tar file using the indicated 
  82     compression method. 
  83   
  84  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  85  """ 
  86   
  87  ######################################################################## 
  88  # Imported modules 
  89  ######################################################################## 
  90   
  91  # System modules 
  92  import os 
  93  import logging 
  94  import datetime 
  95  import pickle 
  96  import tempfile 
  97  from bz2 import BZ2File 
  98  from gzip import GzipFile 
  99  from functools import total_ordering 
 100   
 101  # Cedar Backup modules 
 102  from CedarBackup3.filesystem import FilesystemList, BackupFileList 
 103  from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode 
 104  from CedarBackup3.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList 
 105  from CedarBackup3.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES 
 106  from CedarBackup3.util import isStartOfWeek, buildNormalizedPath 
 107  from CedarBackup3.util import resolveCommand, executeCommand 
 108  from CedarBackup3.util import ObjectTypeList, UnorderedList, RegexList, encodePath, changeOwnership 
 109   
 110   
 111  ######################################################################## 
 112  # Module-wide constants and variables 
 113  ######################################################################## 
 114   
 115  logger = logging.getLogger("CedarBackup3.log.extend.mbox") 
 116   
 117  GREPMAIL_COMMAND = [ "grepmail", ] 
 118  REVISION_PATH_EXTENSION = "mboxlast" 
119 120 121 ######################################################################## 122 # MboxFile class definition 123 ######################################################################## 124 125 @total_ordering 126 -class MboxFile(object):
127 128 """ 129 Class representing mbox file configuration.. 130 131 The following restrictions exist on data in this class: 132 133 - The absolute path must be absolute. 134 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}. 135 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}. 136 137 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, 138 absolutePath, collectMode, compressMode 139 """ 140
141 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None):
142 """ 143 Constructor for the C{MboxFile} class. 144 145 You should never directly instantiate this class. 146 147 @param absolutePath: Absolute path to an mbox file on disk. 148 @param collectMode: Overridden collect mode for this directory. 149 @param compressMode: Overridden compression mode for this directory. 150 """ 151 self._absolutePath = None 152 self._collectMode = None 153 self._compressMode = None 154 self.absolutePath = absolutePath 155 self.collectMode = collectMode 156 self.compressMode = compressMode
157
158 - def __repr__(self):
159 """ 160 Official string representation for class instance. 161 """ 162 return "MboxFile(%s, %s, %s)" % (self.absolutePath, self.collectMode, self.compressMode)
163
164 - def __str__(self):
165 """ 166 Informal string representation for class instance. 167 """ 168 return self.__repr__()
169
170 - def __eq__(self, other):
171 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 172 return self.__cmp__(other) == 0
173
174 - def __lt__(self, other):
175 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 176 return self.__cmp__(other) < 0
177
178 - def __gt__(self, other):
179 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 180 return self.__cmp__(other) > 0
181
182 - def __cmp__(self, other):
183 """ 184 Original Python 2 comparison operator. 185 @param other: Other object to compare to. 186 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 187 """ 188 if other is None: 189 return 1 190 if self.absolutePath != other.absolutePath: 191 if str(self.absolutePath or "") < str(other.absolutePath or ""): 192 return -1 193 else: 194 return 1 195 if self.collectMode != other.collectMode: 196 if str(self.collectMode or "") < str(other.collectMode or ""): 197 return -1 198 else: 199 return 1 200 if self.compressMode != other.compressMode: 201 if str(self.compressMode or "") < str(other.compressMode or ""): 202 return -1 203 else: 204 return 1 205 return 0
206
207 - def _setAbsolutePath(self, value):
208 """ 209 Property target used to set the absolute path. 210 The value must be an absolute path if it is not C{None}. 211 It does not have to exist on disk at the time of assignment. 212 @raise ValueError: If the value is not an absolute path. 213 @raise ValueError: If the value cannot be encoded properly. 214 """ 215 if value is not None: 216 if not os.path.isabs(value): 217 raise ValueError("Absolute path must be, er, an absolute path.") 218 self._absolutePath = encodePath(value)
219
220 - def _getAbsolutePath(self):
221 """ 222 Property target used to get the absolute path. 223 """ 224 return self._absolutePath
225
226 - def _setCollectMode(self, value):
227 """ 228 Property target used to set the collect mode. 229 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}. 230 @raise ValueError: If the value is not valid. 231 """ 232 if value is not None: 233 if value not in VALID_COLLECT_MODES: 234 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES) 235 self._collectMode = value
236
237 - def _getCollectMode(self):
238 """ 239 Property target used to get the collect mode. 240 """ 241 return self._collectMode
242
243 - def _setCompressMode(self, value):
244 """ 245 Property target used to set the compress mode. 246 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}. 247 @raise ValueError: If the value is not valid. 248 """ 249 if value is not None: 250 if value not in VALID_COMPRESS_MODES: 251 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES) 252 self._compressMode = value
253
254 - def _getCompressMode(self):
255 """ 256 Property target used to get the compress mode. 257 """ 258 return self._compressMode
259 260 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox file.") 261 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox file.") 262 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox file.")
263
264 265 ######################################################################## 266 # MboxDir class definition 267 ######################################################################## 268 269 @total_ordering 270 -class MboxDir(object):
271 272 """ 273 Class representing mbox directory configuration.. 274 275 The following restrictions exist on data in this class: 276 277 - The absolute path must be absolute. 278 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}. 279 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}. 280 281 Unlike collect directory configuration, this is the only place exclusions 282 are allowed (no global exclusions at the <mbox> configuration level). Also, 283 we only allow relative exclusions and there is no configured ignore file. 284 This is because mbox directory backups are not recursive. 285 286 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, 287 absolutePath, collectMode, compressMode, relativeExcludePaths, 288 excludePatterns 289 """ 290
291 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None, 292 relativeExcludePaths=None, excludePatterns=None):
293 """ 294 Constructor for the C{MboxDir} class. 295 296 You should never directly instantiate this class. 297 298 @param absolutePath: Absolute path to a mbox file on disk. 299 @param collectMode: Overridden collect mode for this directory. 300 @param compressMode: Overridden compression mode for this directory. 301 @param relativeExcludePaths: List of relative paths to exclude. 302 @param excludePatterns: List of regular expression patterns to exclude 303 """ 304 self._absolutePath = None 305 self._collectMode = None 306 self._compressMode = None 307 self._relativeExcludePaths = None 308 self._excludePatterns = None 309 self.absolutePath = absolutePath 310 self.collectMode = collectMode 311 self.compressMode = compressMode 312 self.relativeExcludePaths = relativeExcludePaths 313 self.excludePatterns = excludePatterns
314
315 - def __repr__(self):
316 """ 317 Official string representation for class instance. 318 """ 319 return "MboxDir(%s, %s, %s, %s, %s)" % (self.absolutePath, self.collectMode, self.compressMode, 320 self.relativeExcludePaths, self.excludePatterns)
321
322 - def __str__(self):
323 """ 324 Informal string representation for class instance. 325 """ 326 return self.__repr__()
327
328 - def __eq__(self, other):
329 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 330 return self.__cmp__(other) == 0
331
332 - def __lt__(self, other):
333 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 334 return self.__cmp__(other) < 0
335
336 - def __gt__(self, other):
337 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 338 return self.__cmp__(other) > 0
339
340 - def __cmp__(self, other):
341 """ 342 Original Python 2 comparison operator. 343 @param other: Other object to compare to. 344 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 345 """ 346 if other is None: 347 return 1 348 if self.absolutePath != other.absolutePath: 349 if str(self.absolutePath or "") < str(other.absolutePath or ""): 350 return -1 351 else: 352 return 1 353 if self.collectMode != other.collectMode: 354 if str(self.collectMode or "") < str(other.collectMode or ""): 355 return -1 356 else: 357 return 1 358 if self.compressMode != other.compressMode: 359 if str(self.compressMode or "") < str(other.compressMode or ""): 360 return -1 361 else: 362 return 1 363 if self.relativeExcludePaths != other.relativeExcludePaths: 364 if self.relativeExcludePaths < other.relativeExcludePaths: 365 return -1 366 else: 367 return 1 368 if self.excludePatterns != other.excludePatterns: 369 if self.excludePatterns < other.excludePatterns: 370 return -1 371 else: 372 return 1 373 return 0
374
375 - def _setAbsolutePath(self, value):
376 """ 377 Property target used to set the absolute path. 378 The value must be an absolute path if it is not C{None}. 379 It does not have to exist on disk at the time of assignment. 380 @raise ValueError: If the value is not an absolute path. 381 @raise ValueError: If the value cannot be encoded properly. 382 """ 383 if value is not None: 384 if not os.path.isabs(value): 385 raise ValueError("Absolute path must be, er, an absolute path.") 386 self._absolutePath = encodePath(value)
387
388 - def _getAbsolutePath(self):
389 """ 390 Property target used to get the absolute path. 391 """ 392 return self._absolutePath
393
394 - def _setCollectMode(self, value):
395 """ 396 Property target used to set the collect mode. 397 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}. 398 @raise ValueError: If the value is not valid. 399 """ 400 if value is not None: 401 if value not in VALID_COLLECT_MODES: 402 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES) 403 self._collectMode = value
404
405 - def _getCollectMode(self):
406 """ 407 Property target used to get the collect mode. 408 """ 409 return self._collectMode
410
411 - def _setCompressMode(self, value):
412 """ 413 Property target used to set the compress mode. 414 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}. 415 @raise ValueError: If the value is not valid. 416 """ 417 if value is not None: 418 if value not in VALID_COMPRESS_MODES: 419 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES) 420 self._compressMode = value
421
422 - def _getCompressMode(self):
423 """ 424 Property target used to get the compress mode. 425 """ 426 return self._compressMode
427
428 - def _setRelativeExcludePaths(self, value):
429 """ 430 Property target used to set the relative exclude paths list. 431 Elements do not have to exist on disk at the time of assignment. 432 """ 433 if value is None: 434 self._relativeExcludePaths = None 435 else: 436 try: 437 saved = self._relativeExcludePaths 438 self._relativeExcludePaths = UnorderedList() 439 self._relativeExcludePaths.extend(value) 440 except Exception as e: 441 self._relativeExcludePaths = saved 442 raise e
443
444 - def _getRelativeExcludePaths(self):
445 """ 446 Property target used to get the relative exclude paths list. 447 """ 448 return self._relativeExcludePaths
449
450 - def _setExcludePatterns(self, value):
451 """ 452 Property target used to set the exclude patterns list. 453 """ 454 if value is None: 455 self._excludePatterns = None 456 else: 457 try: 458 saved = self._excludePatterns 459 self._excludePatterns = RegexList() 460 self._excludePatterns.extend(value) 461 except Exception as e: 462 self._excludePatterns = saved 463 raise e
464
465 - def _getExcludePatterns(self):
466 """ 467 Property target used to get the exclude patterns list. 468 """ 469 return self._excludePatterns
470 471 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox directory.") 472 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox directory.") 473 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox directory.") 474 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.") 475 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
476
477 478 ######################################################################## 479 # MboxConfig class definition 480 ######################################################################## 481 482 @total_ordering 483 -class MboxConfig(object):
484 485 """ 486 Class representing mbox configuration. 487 488 Mbox configuration is used for backing up mbox email files. 489 490 The following restrictions exist on data in this class: 491 492 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}. 493 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}. 494 - The C{mboxFiles} list must be a list of C{MboxFile} objects 495 - The C{mboxDirs} list must be a list of C{MboxDir} objects 496 497 For the C{mboxFiles} and C{mboxDirs} lists, validation is accomplished 498 through the L{util.ObjectTypeList} list implementation that overrides common 499 list methods and transparently ensures that each element is of the proper 500 type. 501 502 Unlike collect configuration, no global exclusions are allowed on this 503 level. We only allow relative exclusions at the mbox directory level. 504 Also, there is no configured ignore file. This is because mbox directory 505 backups are not recursive. 506 507 @note: Lists within this class are "unordered" for equality comparisons. 508 509 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, 510 collectMode, compressMode, mboxFiles, mboxDirs 511 """ 512
513 - def __init__(self, collectMode=None, compressMode=None, mboxFiles=None, mboxDirs=None):
514 """ 515 Constructor for the C{MboxConfig} class. 516 517 @param collectMode: Default collect mode. 518 @param compressMode: Default compress mode. 519 @param mboxFiles: List of mbox files to back up 520 @param mboxDirs: List of mbox directories to back up 521 522 @raise ValueError: If one of the values is invalid. 523 """ 524 self._collectMode = None 525 self._compressMode = None 526 self._mboxFiles = None 527 self._mboxDirs = None 528 self.collectMode = collectMode 529 self.compressMode = compressMode 530 self.mboxFiles = mboxFiles 531 self.mboxDirs = mboxDirs
532
533 - def __repr__(self):
534 """ 535 Official string representation for class instance. 536 """ 537 return "MboxConfig(%s, %s, %s, %s)" % (self.collectMode, self.compressMode, self.mboxFiles, self.mboxDirs)
538
539 - def __str__(self):
540 """ 541 Informal string representation for class instance. 542 """ 543 return self.__repr__()
544
545 - def __eq__(self, other):
546 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 547 return self.__cmp__(other) == 0
548
549 - def __lt__(self, other):
550 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 551 return self.__cmp__(other) < 0
552
553 - def __gt__(self, other):
554 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 555 return self.__cmp__(other) > 0
556
557 - def __cmp__(self, other):
558 """ 559 Original Python 2 comparison operator. 560 Lists within this class are "unordered" for equality comparisons. 561 @param other: Other object to compare to. 562 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 563 """ 564 if other is None: 565 return 1 566 if self.collectMode != other.collectMode: 567 if str(self.collectMode or "") < str(other.collectMode or ""): 568 return -1 569 else: 570 return 1 571 if self.compressMode != other.compressMode: 572 if str(self.compressMode or "") < str(other.compressMode or ""): 573 return -1 574 else: 575 return 1 576 if self.mboxFiles != other.mboxFiles: 577 if self.mboxFiles < other.mboxFiles: 578 return -1 579 else: 580 return 1 581 if self.mboxDirs != other.mboxDirs: 582 if self.mboxDirs < other.mboxDirs: 583 return -1 584 else: 585 return 1 586 return 0
587
588 - def _setCollectMode(self, value):
589 """ 590 Property target used to set the collect mode. 591 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}. 592 @raise ValueError: If the value is not valid. 593 """ 594 if value is not None: 595 if value not in VALID_COLLECT_MODES: 596 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES) 597 self._collectMode = value
598
599 - def _getCollectMode(self):
600 """ 601 Property target used to get the collect mode. 602 """ 603 return self._collectMode
604
605 - def _setCompressMode(self, value):
606 """ 607 Property target used to set the compress mode. 608 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}. 609 @raise ValueError: If the value is not valid. 610 """ 611 if value is not None: 612 if value not in VALID_COMPRESS_MODES: 613 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES) 614 self._compressMode = value
615
616 - def _getCompressMode(self):
617 """ 618 Property target used to get the compress mode. 619 """ 620 return self._compressMode
621
622 - def _setMboxFiles(self, value):
623 """ 624 Property target used to set the mboxFiles list. 625 Either the value must be C{None} or each element must be an C{MboxFile}. 626 @raise ValueError: If the value is not an C{MboxFile} 627 """ 628 if value is None: 629 self._mboxFiles = None 630 else: 631 try: 632 saved = self._mboxFiles 633 self._mboxFiles = ObjectTypeList(MboxFile, "MboxFile") 634 self._mboxFiles.extend(value) 635 except Exception as e: 636 self._mboxFiles = saved 637 raise e
638
639 - def _getMboxFiles(self):
640 """ 641 Property target used to get the mboxFiles list. 642 """ 643 return self._mboxFiles
644
645 - def _setMboxDirs(self, value):
646 """ 647 Property target used to set the mboxDirs list. 648 Either the value must be C{None} or each element must be an C{MboxDir}. 649 @raise ValueError: If the value is not an C{MboxDir} 650 """ 651 if value is None: 652 self._mboxDirs = None 653 else: 654 try: 655 saved = self._mboxDirs 656 self._mboxDirs = ObjectTypeList(MboxDir, "MboxDir") 657 self._mboxDirs.extend(value) 658 except Exception as e: 659 self._mboxDirs = saved 660 raise e
661
662 - def _getMboxDirs(self):
663 """ 664 Property target used to get the mboxDirs list. 665 """ 666 return self._mboxDirs
667 668 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.") 669 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.") 670 mboxFiles = property(_getMboxFiles, _setMboxFiles, None, doc="List of mbox files to back up.") 671 mboxDirs = property(_getMboxDirs, _setMboxDirs, None, doc="List of mbox directories to back up.")
672
673 674 ######################################################################## 675 # LocalConfig class definition 676 ######################################################################## 677 678 @total_ordering 679 -class LocalConfig(object):
680 681 """ 682 Class representing this extension's configuration document. 683 684 This is not a general-purpose configuration object like the main Cedar 685 Backup configuration object. Instead, it just knows how to parse and emit 686 Mbox-specific configuration values. Third parties who need to read and 687 write configuration related to this extension should access it through the 688 constructor, C{validate} and C{addConfig} methods. 689 690 @note: Lists within this class are "unordered" for equality comparisons. 691 692 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, mbox, 693 validate, addConfig 694 """ 695
696 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
697 """ 698 Initializes a configuration object. 699 700 If you initialize the object without passing either C{xmlData} or 701 C{xmlPath} then configuration will be empty and will be invalid until it 702 is filled in properly. 703 704 No reference to the original XML data or original path is saved off by 705 this class. Once the data has been parsed (successfully or not) this 706 original information is discarded. 707 708 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate} 709 method will be called (with its default arguments) against configuration 710 after successfully parsing any passed-in XML. Keep in mind that even if 711 C{validate} is C{False}, it might not be possible to parse the passed-in 712 XML document if lower-level validations fail. 713 714 @note: It is strongly suggested that the C{validate} option always be set 715 to C{True} (the default) unless there is a specific need to read in 716 invalid configuration from disk. 717 718 @param xmlData: XML data representing configuration. 719 @type xmlData: String data. 720 721 @param xmlPath: Path to an XML file on disk. 722 @type xmlPath: Absolute path to a file on disk. 723 724 @param validate: Validate the document after parsing it. 725 @type validate: Boolean true/false. 726 727 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in. 728 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed. 729 @raise ValueError: If the parsed configuration document is not valid. 730 """ 731 self._mbox = None 732 self.mbox = None 733 if xmlData is not None and xmlPath is not None: 734 raise ValueError("Use either xmlData or xmlPath, but not both.") 735 if xmlData is not None: 736 self._parseXmlData(xmlData) 737 if validate: 738 self.validate() 739 elif xmlPath is not None: 740 with open(xmlPath) as f: 741 xmlData = f.read() 742 self._parseXmlData(xmlData) 743 if validate: 744 self.validate()
745
746 - def __repr__(self):
747 """ 748 Official string representation for class instance. 749 """ 750 return "LocalConfig(%s)" % (self.mbox)
751
752 - def __str__(self):
753 """ 754 Informal string representation for class instance. 755 """ 756 return self.__repr__()
757
758 - def __eq__(self, other):
759 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 760 return self.__cmp__(other) == 0
761
762 - def __lt__(self, other):
763 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 764 return self.__cmp__(other) < 0
765
766 - def __gt__(self, other):
767 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 768 return self.__cmp__(other) > 0
769
770 - def __cmp__(self, other):
771 """ 772 Original Python 2 comparison operator. 773 Lists within this class are "unordered" for equality comparisons. 774 @param other: Other object to compare to. 775 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 776 """ 777 if other is None: 778 return 1 779 if self.mbox != other.mbox: 780 if self.mbox < other.mbox: 781 return -1 782 else: 783 return 1 784 return 0
785
786 - def _setMbox(self, value):
787 """ 788 Property target used to set the mbox configuration value. 789 If not C{None}, the value must be a C{MboxConfig} object. 790 @raise ValueError: If the value is not a C{MboxConfig} 791 """ 792 if value is None: 793 self._mbox = None 794 else: 795 if not isinstance(value, MboxConfig): 796 raise ValueError("Value must be a C{MboxConfig} object.") 797 self._mbox = value
798
799 - def _getMbox(self):
800 """ 801 Property target used to get the mbox configuration value. 802 """ 803 return self._mbox
804 805 mbox = property(_getMbox, _setMbox, None, "Mbox configuration in terms of a C{MboxConfig} object.") 806
807 - def validate(self):
808 """ 809 Validates configuration represented by the object. 810 811 Mbox configuration must be filled in. Within that, the collect mode and 812 compress mode are both optional, but the list of repositories must 813 contain at least one entry. 814 815 Each configured file or directory must contain an absolute path, and then 816 must be either able to take collect mode and compress mode configuration 817 from the parent C{MboxConfig} object, or must set each value on its own. 818 819 @raise ValueError: If one of the validations fails. 820 """ 821 if self.mbox is None: 822 raise ValueError("Mbox section is required.") 823 if (self.mbox.mboxFiles is None or len(self.mbox.mboxFiles) < 1) and \ 824 (self.mbox.mboxDirs is None or len(self.mbox.mboxDirs) < 1): 825 raise ValueError("At least one mbox file or directory must be configured.") 826 if self.mbox.mboxFiles is not None: 827 for mboxFile in self.mbox.mboxFiles: 828 if mboxFile.absolutePath is None: 829 raise ValueError("Each mbox file must set an absolute path.") 830 if self.mbox.collectMode is None and mboxFile.collectMode is None: 831 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox file.") 832 if self.mbox.compressMode is None and mboxFile.compressMode is None: 833 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox file.") 834 if self.mbox.mboxDirs is not None: 835 for mboxDir in self.mbox.mboxDirs: 836 if mboxDir.absolutePath is None: 837 raise ValueError("Each mbox directory must set an absolute path.") 838 if self.mbox.collectMode is None and mboxDir.collectMode is None: 839 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox directory.") 840 if self.mbox.compressMode is None and mboxDir.compressMode is None: 841 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox directory.")
842
843 - def addConfig(self, xmlDom, parentNode):
844 """ 845 Adds an <mbox> configuration section as the next child of a parent. 846 847 Third parties should use this function to write configuration related to 848 this extension. 849 850 We add the following fields to the document:: 851 852 collectMode //cb_config/mbox/collectMode 853 compressMode //cb_config/mbox/compressMode 854 855 We also add groups of the following items, one list element per 856 item:: 857 858 mboxFiles //cb_config/mbox/file 859 mboxDirs //cb_config/mbox/dir 860 861 The mbox files and mbox directories are added by L{_addMboxFile} and 862 L{_addMboxDir}. 863 864 @param xmlDom: DOM tree as from C{impl.createDocument()}. 865 @param parentNode: Parent that the section should be appended to. 866 """ 867 if self.mbox is not None: 868 sectionNode = addContainerNode(xmlDom, parentNode, "mbox") 869 addStringNode(xmlDom, sectionNode, "collect_mode", self.mbox.collectMode) 870 addStringNode(xmlDom, sectionNode, "compress_mode", self.mbox.compressMode) 871 if self.mbox.mboxFiles is not None: 872 for mboxFile in self.mbox.mboxFiles: 873 LocalConfig._addMboxFile(xmlDom, sectionNode, mboxFile) 874 if self.mbox.mboxDirs is not None: 875 for mboxDir in self.mbox.mboxDirs: 876 LocalConfig._addMboxDir(xmlDom, sectionNode, mboxDir)
877
878 - def _parseXmlData(self, xmlData):
879 """ 880 Internal method to parse an XML string into the object. 881 882 This method parses the XML document into a DOM tree (C{xmlDom}) and then 883 calls a static method to parse the mbox configuration section. 884 885 @param xmlData: XML data to be parsed 886 @type xmlData: String data 887 888 @raise ValueError: If the XML cannot be successfully parsed. 889 """ 890 (xmlDom, parentNode) = createInputDom(xmlData) 891 self._mbox = LocalConfig._parseMbox(parentNode)
892 893 @staticmethod
894 - def _parseMbox(parent):
895 """ 896 Parses an mbox configuration section. 897 898 We read the following individual fields:: 899 900 collectMode //cb_config/mbox/collect_mode 901 compressMode //cb_config/mbox/compress_mode 902 903 We also read groups of the following item, one list element per 904 item:: 905 906 mboxFiles //cb_config/mbox/file 907 mboxDirs //cb_config/mbox/dir 908 909 The mbox files are parsed by L{_parseMboxFiles} and the mbox 910 directories are parsed by L{_parseMboxDirs}. 911 912 @param parent: Parent node to search beneath. 913 914 @return: C{MboxConfig} object or C{None} if the section does not exist. 915 @raise ValueError: If some filled-in value is invalid. 916 """ 917 mbox = None 918 section = readFirstChild(parent, "mbox") 919 if section is not None: 920 mbox = MboxConfig() 921 mbox.collectMode = readString(section, "collect_mode") 922 mbox.compressMode = readString(section, "compress_mode") 923 mbox.mboxFiles = LocalConfig._parseMboxFiles(section) 924 mbox.mboxDirs = LocalConfig._parseMboxDirs(section) 925 return mbox
926 927 @staticmethod
928 - def _parseMboxFiles(parent):
929 """ 930 Reads a list of C{MboxFile} objects from immediately beneath the parent. 931 932 We read the following individual fields:: 933 934 absolutePath abs_path 935 collectMode collect_mode 936 compressMode compess_mode 937 938 @param parent: Parent node to search beneath. 939 940 @return: List of C{MboxFile} objects or C{None} if none are found. 941 @raise ValueError: If some filled-in value is invalid. 942 """ 943 lst = [] 944 for entry in readChildren(parent, "file"): 945 if isElement(entry): 946 mboxFile = MboxFile() 947 mboxFile.absolutePath = readString(entry, "abs_path") 948 mboxFile.collectMode = readString(entry, "collect_mode") 949 mboxFile.compressMode = readString(entry, "compress_mode") 950 lst.append(mboxFile) 951 if lst == []: 952 lst = None 953 return lst
954 955 @staticmethod
956 - def _parseMboxDirs(parent):
957 """ 958 Reads a list of C{MboxDir} objects from immediately beneath the parent. 959 960 We read the following individual fields:: 961 962 absolutePath abs_path 963 collectMode collect_mode 964 compressMode compess_mode 965 966 We also read groups of the following items, one list element per 967 item:: 968 969 relativeExcludePaths exclude/rel_path 970 excludePatterns exclude/pattern 971 972 The exclusions are parsed by L{_parseExclusions}. 973 974 @param parent: Parent node to search beneath. 975 976 @return: List of C{MboxDir} objects or C{None} if none are found. 977 @raise ValueError: If some filled-in value is invalid. 978 """ 979 lst = [] 980 for entry in readChildren(parent, "dir"): 981 if isElement(entry): 982 mboxDir = MboxDir() 983 mboxDir.absolutePath = readString(entry, "abs_path") 984 mboxDir.collectMode = readString(entry, "collect_mode") 985 mboxDir.compressMode = readString(entry, "compress_mode") 986 (mboxDir.relativeExcludePaths, mboxDir.excludePatterns) = LocalConfig._parseExclusions(entry) 987 lst.append(mboxDir) 988 if lst == []: 989 lst = None 990 return lst
991 992 @staticmethod
993 - def _parseExclusions(parentNode):
994 """ 995 Reads exclusions data from immediately beneath the parent. 996 997 We read groups of the following items, one list element per item:: 998 999 relative exclude/rel_path 1000 patterns exclude/pattern 1001 1002 If there are none of some pattern (i.e. no relative path items) then 1003 C{None} will be returned for that item in the tuple. 1004 1005 @param parentNode: Parent node to search beneath. 1006 1007 @return: Tuple of (relative, patterns) exclusions. 1008 """ 1009 section = readFirstChild(parentNode, "exclude") 1010 if section is None: 1011 return (None, None) 1012 else: 1013 relative = readStringList(section, "rel_path") 1014 patterns = readStringList(section, "pattern") 1015 return (relative, patterns)
1016 1017 @staticmethod
1018 - def _addMboxFile(xmlDom, parentNode, mboxFile):
1019 """ 1020 Adds an mbox file container as the next child of a parent. 1021 1022 We add the following fields to the document:: 1023 1024 absolutePath file/abs_path 1025 collectMode file/collect_mode 1026 compressMode file/compress_mode 1027 1028 The <file> node itself is created as the next child of the parent node. 1029 This method only adds one mbox file node. The parent must loop for each 1030 mbox file in the C{MboxConfig} object. 1031 1032 If C{mboxFile} is C{None}, this method call will be a no-op. 1033 1034 @param xmlDom: DOM tree as from C{impl.createDocument()}. 1035 @param parentNode: Parent that the section should be appended to. 1036 @param mboxFile: MboxFile to be added to the document. 1037 """ 1038 if mboxFile is not None: 1039 sectionNode = addContainerNode(xmlDom, parentNode, "file") 1040 addStringNode(xmlDom, sectionNode, "abs_path", mboxFile.absolutePath) 1041 addStringNode(xmlDom, sectionNode, "collect_mode", mboxFile.collectMode) 1042 addStringNode(xmlDom, sectionNode, "compress_mode", mboxFile.compressMode)
1043 1044 @staticmethod
1045 - def _addMboxDir(xmlDom, parentNode, mboxDir):
1046 """ 1047 Adds an mbox directory container as the next child of a parent. 1048 1049 We add the following fields to the document:: 1050 1051 absolutePath dir/abs_path 1052 collectMode dir/collect_mode 1053 compressMode dir/compress_mode 1054 1055 We also add groups of the following items, one list element per item:: 1056 1057 relativeExcludePaths dir/exclude/rel_path 1058 excludePatterns dir/exclude/pattern 1059 1060 The <dir> node itself is created as the next child of the parent node. 1061 This method only adds one mbox directory node. The parent must loop for 1062 each mbox directory in the C{MboxConfig} object. 1063 1064 If C{mboxDir} is C{None}, this method call will be a no-op. 1065 1066 @param xmlDom: DOM tree as from C{impl.createDocument()}. 1067 @param parentNode: Parent that the section should be appended to. 1068 @param mboxDir: MboxDir to be added to the document. 1069 """ 1070 if mboxDir is not None: 1071 sectionNode = addContainerNode(xmlDom, parentNode, "dir") 1072 addStringNode(xmlDom, sectionNode, "abs_path", mboxDir.absolutePath) 1073 addStringNode(xmlDom, sectionNode, "collect_mode", mboxDir.collectMode) 1074 addStringNode(xmlDom, sectionNode, "compress_mode", mboxDir.compressMode) 1075 if ((mboxDir.relativeExcludePaths is not None and mboxDir.relativeExcludePaths != []) or 1076 (mboxDir.excludePatterns is not None and mboxDir.excludePatterns != [])): 1077 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude") 1078 if mboxDir.relativeExcludePaths is not None: 1079 for relativePath in mboxDir.relativeExcludePaths: 1080 addStringNode(xmlDom, excludeNode, "rel_path", relativePath) 1081 if mboxDir.excludePatterns is not None: 1082 for pattern in mboxDir.excludePatterns: 1083 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1084
1085 1086 ######################################################################## 1087 # Public functions 1088 ######################################################################## 1089 1090 ########################### 1091 # executeAction() function 1092 ########################### 1093 1094 -def executeAction(configPath, options, config):
1095 """ 1096 Executes the mbox backup action. 1097 1098 @param configPath: Path to configuration file on disk. 1099 @type configPath: String representing a path on disk. 1100 1101 @param options: Program command-line options. 1102 @type options: Options object. 1103 1104 @param config: Program configuration. 1105 @type config: Config object. 1106 1107 @raise ValueError: Under many generic error conditions 1108 @raise IOError: If a backup could not be written for some reason. 1109 """ 1110 logger.debug("Executing mbox extended action.") 1111 newRevision = datetime.datetime.today() # mark here so all actions are after this date/time 1112 if config.options is None or config.collect is None: 1113 raise ValueError("Cedar Backup configuration is not properly filled in.") 1114 local = LocalConfig(xmlPath=configPath) 1115 todayIsStart = isStartOfWeek(config.options.startingDay) 1116 fullBackup = options.full or todayIsStart 1117 logger.debug("Full backup flag is [%s]", fullBackup) 1118 if local.mbox.mboxFiles is not None: 1119 for mboxFile in local.mbox.mboxFiles: 1120 logger.debug("Working with mbox file [%s]", mboxFile.absolutePath) 1121 collectMode = _getCollectMode(local, mboxFile) 1122 compressMode = _getCompressMode(local, mboxFile) 1123 lastRevision = _loadLastRevision(config, mboxFile, fullBackup, collectMode) 1124 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart): 1125 logger.debug("Mbox file meets criteria to be backed up today.") 1126 _backupMboxFile(config, mboxFile.absolutePath, fullBackup, 1127 collectMode, compressMode, lastRevision, newRevision) 1128 else: 1129 logger.debug("Mbox file will not be backed up, per collect mode.") 1130 if collectMode == 'incr': 1131 _writeNewRevision(config, mboxFile, newRevision) 1132 if local.mbox.mboxDirs is not None: 1133 for mboxDir in local.mbox.mboxDirs: 1134 logger.debug("Working with mbox directory [%s]", mboxDir.absolutePath) 1135 collectMode = _getCollectMode(local, mboxDir) 1136 compressMode = _getCompressMode(local, mboxDir) 1137 lastRevision = _loadLastRevision(config, mboxDir, fullBackup, collectMode) 1138 (excludePaths, excludePatterns) = _getExclusions(mboxDir) 1139 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart): 1140 logger.debug("Mbox directory meets criteria to be backed up today.") 1141 _backupMboxDir(config, mboxDir.absolutePath, 1142 fullBackup, collectMode, compressMode, 1143 lastRevision, newRevision, 1144 excludePaths, excludePatterns) 1145 else: 1146 logger.debug("Mbox directory will not be backed up, per collect mode.") 1147 if collectMode == 'incr': 1148 _writeNewRevision(config, mboxDir, newRevision) 1149 logger.info("Executed the mbox extended action successfully.")
1150
1151 -def _getCollectMode(local, item):
1152 """ 1153 Gets the collect mode that should be used for an mbox file or directory. 1154 Use file- or directory-specific value if possible, otherwise take from mbox section. 1155 @param local: LocalConfig object. 1156 @param item: Mbox file or directory 1157 @return: Collect mode to use. 1158 """ 1159 if item.collectMode is None: 1160 collectMode = local.mbox.collectMode 1161 else: 1162 collectMode = item.collectMode 1163 logger.debug("Collect mode is [%s]", collectMode) 1164 return collectMode
1165
1166 -def _getCompressMode(local, item):
1167 """ 1168 Gets the compress mode that should be used for an mbox file or directory. 1169 Use file- or directory-specific value if possible, otherwise take from mbox section. 1170 @param local: LocalConfig object. 1171 @param item: Mbox file or directory 1172 @return: Compress mode to use. 1173 """ 1174 if item.compressMode is None: 1175 compressMode = local.mbox.compressMode 1176 else: 1177 compressMode = item.compressMode 1178 logger.debug("Compress mode is [%s]", compressMode) 1179 return compressMode
1180
1181 -def _getRevisionPath(config, item):
1182 """ 1183 Gets the path to the revision file associated with a repository. 1184 @param config: Cedar Backup configuration. 1185 @param item: Mbox file or directory 1186 @return: Absolute path to the revision file associated with the repository. 1187 """ 1188 normalized = buildNormalizedPath(item.absolutePath) 1189 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION) 1190 revisionPath = os.path.join(config.options.workingDir, filename) 1191 logger.debug("Revision file path is [%s]", revisionPath) 1192 return revisionPath
1193
1194 -def _loadLastRevision(config, item, fullBackup, collectMode):
1195 """ 1196 Loads the last revision date for this item from disk and returns it. 1197 1198 If this is a full backup, or if the revision file cannot be loaded for some 1199 reason, then C{None} is returned. This indicates that there is no previous 1200 revision, so the entire mail file or directory should be backed up. 1201 1202 @note: We write the actual revision object to disk via pickle, so we don't 1203 deal with the datetime precision or format at all. Whatever's in the object 1204 is what we write. 1205 1206 @param config: Cedar Backup configuration. 1207 @param item: Mbox file or directory 1208 @param fullBackup: Indicates whether this is a full backup 1209 @param collectMode: Indicates the collect mode for this item 1210 1211 @return: Revision date as a datetime.datetime object or C{None}. 1212 """ 1213 revisionPath = _getRevisionPath(config, item) 1214 if fullBackup: 1215 revisionDate = None 1216 logger.debug("Revision file ignored because this is a full backup.") 1217 elif collectMode in ['weekly', 'daily']: 1218 revisionDate = None 1219 logger.debug("No revision file based on collect mode [%s].", collectMode) 1220 else: 1221 logger.debug("Revision file will be used for non-full incremental backup.") 1222 if not os.path.isfile(revisionPath): 1223 revisionDate = None 1224 logger.debug("Revision file [%s] does not exist on disk.", revisionPath) 1225 else: 1226 try: 1227 with open(revisionPath, "rb") as f: 1228 revisionDate = pickle.load(f, fix_imports=True) # be compatible with Python 2 1229 logger.debug("Loaded revision file [%s] from disk: [%s]", revisionPath, revisionDate) 1230 except Exception as e: 1231 revisionDate = None 1232 logger.error("Failed loading revision file [%s] from disk: %s", revisionPath, e) 1233 return revisionDate
1234
1235 -def _writeNewRevision(config, item, newRevision):
1236 """ 1237 Writes new revision information to disk. 1238 1239 If we can't write the revision file successfully for any reason, we'll log 1240 the condition but won't throw an exception. 1241 1242 @note: We write the actual revision object to disk via pickle, so we don't 1243 deal with the datetime precision or format at all. Whatever's in the object 1244 is what we write. 1245 1246 @param config: Cedar Backup configuration. 1247 @param item: Mbox file or directory 1248 @param newRevision: Revision date as a datetime.datetime object. 1249 """ 1250 revisionPath = _getRevisionPath(config, item) 1251 try: 1252 with open(revisionPath, "wb") as f: 1253 pickle.dump(newRevision, f, 0, fix_imports=True) # be compatible with Python 2 1254 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup) 1255 logger.debug("Wrote new revision file [%s] to disk: [%s]", revisionPath, newRevision) 1256 except Exception as e: 1257 logger.error("Failed to write revision file [%s] to disk: %s", revisionPath, e)
1258
1259 -def _getExclusions(mboxDir):
1260 """ 1261 Gets exclusions (file and patterns) associated with an mbox directory. 1262 1263 The returned files value is a list of absolute paths to be excluded from the 1264 backup for a given directory. It is derived from the mbox directory's 1265 relative exclude paths. 1266 1267 The returned patterns value is a list of patterns to be excluded from the 1268 backup for a given directory. It is derived from the mbox directory's list 1269 of patterns. 1270 1271 @param mboxDir: Mbox directory object. 1272 1273 @return: Tuple (files, patterns) indicating what to exclude. 1274 """ 1275 paths = [] 1276 if mboxDir.relativeExcludePaths is not None: 1277 for relativePath in mboxDir.relativeExcludePaths: 1278 paths.append(os.path.join(mboxDir.absolutePath, relativePath)) 1279 patterns = [] 1280 if mboxDir.excludePatterns is not None: 1281 patterns.extend(mboxDir.excludePatterns) 1282 logger.debug("Exclude paths: %s", paths) 1283 logger.debug("Exclude patterns: %s", patterns) 1284 return(paths, patterns)
1285
1286 -def _getBackupPath(config, mboxPath, compressMode, newRevision, targetDir=None):
1287 """ 1288 Gets the backup file path (including correct extension) associated with an mbox path. 1289 1290 We assume that if the target directory is passed in, that we're backing up a 1291 directory. Under these circumstances, we'll just use the basename of the 1292 individual path as the output file. 1293 1294 @note: The backup path only contains the current date in YYYYMMDD format, 1295 but that's OK because the index information (stored elsewhere) is the actual 1296 date object. 1297 1298 @param config: Cedar Backup configuration. 1299 @param mboxPath: Path to the indicated mbox file or directory 1300 @param compressMode: Compress mode to use for this mbox path 1301 @param newRevision: Revision this backup path represents 1302 @param targetDir: Target directory in which the path should exist 1303 1304 @return: Absolute path to the backup file associated with the repository. 1305 """ 1306 if targetDir is None: 1307 normalizedPath = buildNormalizedPath(mboxPath) 1308 revisionDate = newRevision.strftime("%Y%m%d") 1309 filename = "mbox-%s-%s" % (revisionDate, normalizedPath) 1310 else: 1311 filename = os.path.basename(mboxPath) 1312 if compressMode == 'gzip': 1313 filename = "%s.gz" % filename 1314 elif compressMode == 'bzip2': 1315 filename = "%s.bz2" % filename 1316 if targetDir is None: 1317 backupPath = os.path.join(config.collect.targetDir, filename) 1318 else: 1319 backupPath = os.path.join(targetDir, filename) 1320 logger.debug("Backup file path is [%s]", backupPath) 1321 return backupPath
1322
1323 -def _getTarfilePath(config, mboxPath, compressMode, newRevision):
1324 """ 1325 Gets the tarfile backup file path (including correct extension) associated 1326 with an mbox path. 1327 1328 Along with the path, the tar archive mode is returned in a form that can 1329 be used with L{BackupFileList.generateTarfile}. 1330 1331 @note: The tarfile path only contains the current date in YYYYMMDD format, 1332 but that's OK because the index information (stored elsewhere) is the actual 1333 date object. 1334 1335 @param config: Cedar Backup configuration. 1336 @param mboxPath: Path to the indicated mbox file or directory 1337 @param compressMode: Compress mode to use for this mbox path 1338 @param newRevision: Revision this backup path represents 1339 1340 @return: Tuple of (absolute path to tarfile, tar archive mode) 1341 """ 1342 normalizedPath = buildNormalizedPath(mboxPath) 1343 revisionDate = newRevision.strftime("%Y%m%d") 1344 filename = "mbox-%s-%s.tar" % (revisionDate, normalizedPath) 1345 if compressMode == 'gzip': 1346 filename = "%s.gz" % filename 1347 archiveMode = "targz" 1348 elif compressMode == 'bzip2': 1349 filename = "%s.bz2" % filename 1350 archiveMode = "tarbz2" 1351 else: 1352 archiveMode = "tar" 1353 tarfilePath = os.path.join(config.collect.targetDir, filename) 1354 logger.debug("Tarfile path is [%s]", tarfilePath) 1355 return (tarfilePath, archiveMode)
1356
1357 -def _getOutputFile(backupPath, compressMode):
1358 """ 1359 Opens the output file used for saving backup information. 1360 1361 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the 1362 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just 1363 return an object from the normal C{open()} method. 1364 1365 @param backupPath: Path to file to open. 1366 @param compressMode: Compress mode of file ("none", "gzip", "bzip"). 1367 1368 @return: Output file object, opened in binary mode for use with executeCommand() 1369 """ 1370 if compressMode == "gzip": 1371 return GzipFile(backupPath, "wb") 1372 elif compressMode == "bzip2": 1373 return BZ2File(backupPath, "wb") 1374 else: 1375 return open(backupPath, "wb")
1376
1377 -def _backupMboxFile(config, absolutePath, 1378 fullBackup, collectMode, compressMode, 1379 lastRevision, newRevision, targetDir=None):
1380 """ 1381 Backs up an individual mbox file. 1382 1383 @param config: Cedar Backup configuration. 1384 @param absolutePath: Path to mbox file to back up. 1385 @param fullBackup: Indicates whether this should be a full backup. 1386 @param collectMode: Indicates the collect mode for this item 1387 @param compressMode: Compress mode of file ("none", "gzip", "bzip") 1388 @param lastRevision: Date of last backup as datetime.datetime 1389 @param newRevision: Date of new (current) backup as datetime.datetime 1390 @param targetDir: Target directory to write the backed-up file into 1391 1392 @raise ValueError: If some value is missing or invalid. 1393 @raise IOError: If there is a problem backing up the mbox file. 1394 """ 1395 if fullBackup or collectMode != "incr" or lastRevision is None: 1396 args = [ "-a", "-u", absolutePath, ] # remove duplicates but fetch entire mailbox 1397 else: 1398 revisionDate = lastRevision.strftime("%Y-%m-%dT%H:%M:%S") # ISO-8601 format; grepmail calls Date::Parse::str2time() 1399 args = [ "-a", "-u", "-d", "since %s" % revisionDate, absolutePath, ] 1400 command = resolveCommand(GREPMAIL_COMMAND) 1401 backupPath = _getBackupPath(config, absolutePath, compressMode, newRevision, targetDir=targetDir) 1402 with _getOutputFile(backupPath, compressMode) as outputFile: 1403 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=outputFile)[0] 1404 if result != 0: 1405 raise IOError("Error [%d] executing grepmail on [%s]." % (result, absolutePath)) 1406 logger.debug("Completed backing up mailbox [%s].", absolutePath) 1407 return backupPath
1408
1409 -def _backupMboxDir(config, absolutePath, 1410 fullBackup, collectMode, compressMode, 1411 lastRevision, newRevision, 1412 excludePaths, excludePatterns):
1413 """ 1414 Backs up a directory containing mbox files. 1415 1416 @param config: Cedar Backup configuration. 1417 @param absolutePath: Path to mbox directory to back up. 1418 @param fullBackup: Indicates whether this should be a full backup. 1419 @param collectMode: Indicates the collect mode for this item 1420 @param compressMode: Compress mode of file ("none", "gzip", "bzip") 1421 @param lastRevision: Date of last backup as datetime.datetime 1422 @param newRevision: Date of new (current) backup as datetime.datetime 1423 @param excludePaths: List of absolute paths to exclude. 1424 @param excludePatterns: List of patterns to exclude. 1425 1426 @raise ValueError: If some value is missing or invalid. 1427 @raise IOError: If there is a problem backing up the mbox file. 1428 """ 1429 try: 1430 tmpdir = tempfile.mkdtemp(dir=config.options.workingDir) 1431 mboxList = FilesystemList() 1432 mboxList.excludeDirs = True 1433 mboxList.excludePaths = excludePaths 1434 mboxList.excludePatterns = excludePatterns 1435 mboxList.addDirContents(absolutePath, recursive=False) 1436 tarList = BackupFileList() 1437 for item in mboxList: 1438 backupPath = _backupMboxFile(config, item, fullBackup, 1439 collectMode, "none", # no need to compress inside compressed tar 1440 lastRevision, newRevision, 1441 targetDir=tmpdir) 1442 tarList.addFile(backupPath) 1443 (tarfilePath, archiveMode) = _getTarfilePath(config, absolutePath, compressMode, newRevision) 1444 tarList.generateTarfile(tarfilePath, archiveMode, ignore=True, flat=True) 1445 changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup) 1446 logger.debug("Completed backing up directory [%s].", absolutePath) 1447 finally: 1448 try: 1449 for cleanitem in tarList: 1450 if os.path.exists(cleanitem): 1451 try: 1452 os.remove(cleanitem) 1453 except: pass 1454 except: pass 1455 try: 1456 os.rmdir(tmpdir) 1457 except: pass
1458