Package CedarBackup3 :: Package extend :: Module split
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup3.extend.split

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2007,2010,2013,2015 Kenneth J. Pronovici. 
 12  # All rights reserved. 
 13  # 
 14  # This program is free software; you can redistribute it and/or 
 15  # modify it under the terms of the GNU General Public License, 
 16  # Version 2, as published by the Free Software Foundation. 
 17  # 
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 21  # 
 22  # Copies of the GNU General Public License are available from 
 23  # the Free Software Foundation website, http://www.gnu.org/. 
 24  # 
 25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 26  # 
 27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 28  # Language : Python 3 (>= 3.4) 
 29  # Project  : Official Cedar Backup Extensions 
 30  # Purpose  : Provides an extension to split up large files in staging directories. 
 31  # 
 32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 33   
 34  ######################################################################## 
 35  # Module documentation 
 36  ######################################################################## 
 37   
 38  """ 
 39  Provides an extension to split up large files in staging directories. 
 40   
 41  When this extension is executed, it will look through the configured Cedar 
 42  Backup staging directory for files exceeding a specified size limit, and split 
 43  them down into smaller files using the 'split' utility.  Any directory which 
 44  has already been split (as indicated by the C{cback.split} file) will be 
 45  ignored. 
 46   
 47  This extension requires a new configuration section <split> and is intended 
 48  to be run immediately after the standard stage action or immediately before the 
 49  standard store action.  Aside from its own configuration, it requires the 
 50  options and staging configuration sections in the standard Cedar Backup 
 51  configuration file. 
 52   
 53  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 54  """ 
 55   
 56  ######################################################################## 
 57  # Imported modules 
 58  ######################################################################## 
 59   
 60  # System modules 
 61  import os 
 62  import re 
 63  import logging 
 64  from functools import total_ordering 
 65   
 66  # Cedar Backup modules 
 67  from CedarBackup3.util import resolveCommand, executeCommand, changeOwnership 
 68  from CedarBackup3.xmlutil import createInputDom, addContainerNode 
 69  from CedarBackup3.xmlutil import readFirstChild 
 70  from CedarBackup3.actions.util import findDailyDirs, writeIndicatorFile, getBackupFiles 
 71  from CedarBackup3.config import ByteQuantity, readByteQuantity, addByteQuantityNode 
 72   
 73   
 74  ######################################################################## 
 75  # Module-wide constants and variables 
 76  ######################################################################## 
 77   
 78  logger = logging.getLogger("CedarBackup3.log.extend.split") 
 79   
 80  SPLIT_COMMAND = [ "split", ] 
 81  SPLIT_INDICATOR = "cback.split" 
82 83 84 ######################################################################## 85 # SplitConfig class definition 86 ######################################################################## 87 88 @total_ordering 89 -class SplitConfig(object):
90 91 """ 92 Class representing split configuration. 93 94 Split configuration is used for splitting staging directories. 95 96 The following restrictions exist on data in this class: 97 98 - The size limit must be a ByteQuantity 99 - The split size must be a ByteQuantity 100 101 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, 102 sizeLimit, splitSize 103 """ 104
105 - def __init__(self, sizeLimit=None, splitSize=None):
106 """ 107 Constructor for the C{SplitCOnfig} class. 108 109 @param sizeLimit: Size limit of the files, in bytes 110 @param splitSize: Size that files exceeding the limit will be split into, in bytes 111 112 @raise ValueError: If one of the values is invalid. 113 """ 114 self._sizeLimit = None 115 self._splitSize = None 116 self.sizeLimit = sizeLimit 117 self.splitSize = splitSize
118
119 - def __repr__(self):
120 """ 121 Official string representation for class instance. 122 """ 123 return "SplitConfig(%s, %s)" % (self.sizeLimit, self.splitSize)
124
125 - def __str__(self):
126 """ 127 Informal string representation for class instance. 128 """ 129 return self.__repr__()
130
131 - def __eq__(self, other):
132 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 133 return self.__cmp__(other) == 0
134
135 - def __lt__(self, other):
136 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 137 return self.__cmp__(other) < 0
138
139 - def __gt__(self, other):
140 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 141 return self.__cmp__(other) > 0
142
143 - def __cmp__(self, other):
144 """ 145 Original Python 2 comparison operator. 146 Lists within this class are "unordered" for equality comparisons. 147 @param other: Other object to compare to. 148 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 149 """ 150 if other is None: 151 return 1 152 if self.sizeLimit != other.sizeLimit: 153 if (self.sizeLimit or ByteQuantity()) < (other.sizeLimit or ByteQuantity()): 154 return -1 155 else: 156 return 1 157 if self.splitSize != other.splitSize: 158 if (self.splitSize or ByteQuantity()) < (other.splitSize or ByteQuantity()): 159 return -1 160 else: 161 return 1 162 return 0
163
164 - def _setSizeLimit(self, value):
165 """ 166 Property target used to set the size limit. 167 If not C{None}, the value must be a C{ByteQuantity} object. 168 @raise ValueError: If the value is not a C{ByteQuantity} 169 """ 170 if value is None: 171 self._sizeLimit = None 172 else: 173 if not isinstance(value, ByteQuantity): 174 raise ValueError("Value must be a C{ByteQuantity} object.") 175 self._sizeLimit = value
176
177 - def _getSizeLimit(self):
178 """ 179 Property target used to get the size limit. 180 """ 181 return self._sizeLimit
182
183 - def _setSplitSize(self, value):
184 """ 185 Property target used to set the split size. 186 If not C{None}, the value must be a C{ByteQuantity} object. 187 @raise ValueError: If the value is not a C{ByteQuantity} 188 """ 189 if value is None: 190 self._splitSize = None 191 else: 192 if not isinstance(value, ByteQuantity): 193 raise ValueError("Value must be a C{ByteQuantity} object.") 194 self._splitSize = value
195
196 - def _getSplitSize(self):
197 """ 198 Property target used to get the split size. 199 """ 200 return self._splitSize
201 202 sizeLimit = property(_getSizeLimit, _setSizeLimit, None, doc="Size limit, as a ByteQuantity") 203 splitSize = property(_getSplitSize, _setSplitSize, None, doc="Split size, as a ByteQuantity")
204
205 206 ######################################################################## 207 # LocalConfig class definition 208 ######################################################################## 209 210 @total_ordering 211 -class LocalConfig(object):
212 213 """ 214 Class representing this extension's configuration document. 215 216 This is not a general-purpose configuration object like the main Cedar 217 Backup configuration object. Instead, it just knows how to parse and emit 218 split-specific configuration values. Third parties who need to read and 219 write configuration related to this extension should access it through the 220 constructor, C{validate} and C{addConfig} methods. 221 222 @note: Lists within this class are "unordered" for equality comparisons. 223 224 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, split, 225 validate, addConfig 226 """ 227
228 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
229 """ 230 Initializes a configuration object. 231 232 If you initialize the object without passing either C{xmlData} or 233 C{xmlPath} then configuration will be empty and will be invalid until it 234 is filled in properly. 235 236 No reference to the original XML data or original path is saved off by 237 this class. Once the data has been parsed (successfully or not) this 238 original information is discarded. 239 240 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate} 241 method will be called (with its default arguments) against configuration 242 after successfully parsing any passed-in XML. Keep in mind that even if 243 C{validate} is C{False}, it might not be possible to parse the passed-in 244 XML document if lower-level validations fail. 245 246 @note: It is strongly suggested that the C{validate} option always be set 247 to C{True} (the default) unless there is a specific need to read in 248 invalid configuration from disk. 249 250 @param xmlData: XML data representing configuration. 251 @type xmlData: String data. 252 253 @param xmlPath: Path to an XML file on disk. 254 @type xmlPath: Absolute path to a file on disk. 255 256 @param validate: Validate the document after parsing it. 257 @type validate: Boolean true/false. 258 259 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in. 260 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed. 261 @raise ValueError: If the parsed configuration document is not valid. 262 """ 263 self._split = None 264 self.split = None 265 if xmlData is not None and xmlPath is not None: 266 raise ValueError("Use either xmlData or xmlPath, but not both.") 267 if xmlData is not None: 268 self._parseXmlData(xmlData) 269 if validate: 270 self.validate() 271 elif xmlPath is not None: 272 with open(xmlPath) as f: 273 xmlData = f.read() 274 self._parseXmlData(xmlData) 275 if validate: 276 self.validate()
277
278 - def __repr__(self):
279 """ 280 Official string representation for class instance. 281 """ 282 return "LocalConfig(%s)" % (self.split)
283
284 - def __str__(self):
285 """ 286 Informal string representation for class instance. 287 """ 288 return self.__repr__()
289
290 - def __eq__(self, other):
291 """Equals operator, iplemented in terms of original Python 2 compare operator.""" 292 return self.__cmp__(other) == 0
293
294 - def __lt__(self, other):
295 """Less-than operator, iplemented in terms of original Python 2 compare operator.""" 296 return self.__cmp__(other) < 0
297
298 - def __gt__(self, other):
299 """Greater-than operator, iplemented in terms of original Python 2 compare operator.""" 300 return self.__cmp__(other) > 0
301
302 - def __cmp__(self, other):
303 """ 304 Original Python 2 comparison operator. 305 Lists within this class are "unordered" for equality comparisons. 306 @param other: Other object to compare to. 307 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 308 """ 309 if other is None: 310 return 1 311 if self.split != other.split: 312 if self.split < other.split: 313 return -1 314 else: 315 return 1 316 return 0
317
318 - def _setSplit(self, value):
319 """ 320 Property target used to set the split configuration value. 321 If not C{None}, the value must be a C{SplitConfig} object. 322 @raise ValueError: If the value is not a C{SplitConfig} 323 """ 324 if value is None: 325 self._split = None 326 else: 327 if not isinstance(value, SplitConfig): 328 raise ValueError("Value must be a C{SplitConfig} object.") 329 self._split = value
330
331 - def _getSplit(self):
332 """ 333 Property target used to get the split configuration value. 334 """ 335 return self._split
336 337 split = property(_getSplit, _setSplit, None, "Split configuration in terms of a C{SplitConfig} object.") 338
339 - def validate(self):
340 """ 341 Validates configuration represented by the object. 342 343 Split configuration must be filled in. Within that, both the size limit 344 and split size must be filled in. 345 346 @raise ValueError: If one of the validations fails. 347 """ 348 if self.split is None: 349 raise ValueError("Split section is required.") 350 if self.split.sizeLimit is None: 351 raise ValueError("Size limit must be set.") 352 if self.split.splitSize is None: 353 raise ValueError("Split size must be set.")
354
355 - def addConfig(self, xmlDom, parentNode):
356 """ 357 Adds a <split> configuration section as the next child of a parent. 358 359 Third parties should use this function to write configuration related to 360 this extension. 361 362 We add the following fields to the document:: 363 364 sizeLimit //cb_config/split/size_limit 365 splitSize //cb_config/split/split_size 366 367 @param xmlDom: DOM tree as from C{impl.createDocument()}. 368 @param parentNode: Parent that the section should be appended to. 369 """ 370 if self.split is not None: 371 sectionNode = addContainerNode(xmlDom, parentNode, "split") 372 addByteQuantityNode(xmlDom, sectionNode, "size_limit", self.split.sizeLimit) 373 addByteQuantityNode(xmlDom, sectionNode, "split_size", self.split.splitSize)
374
375 - def _parseXmlData(self, xmlData):
376 """ 377 Internal method to parse an XML string into the object. 378 379 This method parses the XML document into a DOM tree (C{xmlDom}) and then 380 calls a static method to parse the split configuration section. 381 382 @param xmlData: XML data to be parsed 383 @type xmlData: String data 384 385 @raise ValueError: If the XML cannot be successfully parsed. 386 """ 387 (xmlDom, parentNode) = createInputDom(xmlData) 388 self._split = LocalConfig._parseSplit(parentNode)
389 390 @staticmethod
391 - def _parseSplit(parent):
392 """ 393 Parses an split configuration section. 394 395 We read the following individual fields:: 396 397 sizeLimit //cb_config/split/size_limit 398 splitSize //cb_config/split/split_size 399 400 @param parent: Parent node to search beneath. 401 402 @return: C{EncryptConfig} object or C{None} if the section does not exist. 403 @raise ValueError: If some filled-in value is invalid. 404 """ 405 split = None 406 section = readFirstChild(parent, "split") 407 if section is not None: 408 split = SplitConfig() 409 split.sizeLimit = readByteQuantity(section, "size_limit") 410 split.splitSize = readByteQuantity(section, "split_size") 411 return split
412
413 414 ######################################################################## 415 # Public functions 416 ######################################################################## 417 418 ########################### 419 # executeAction() function 420 ########################### 421 422 -def executeAction(configPath, options, config):
423 """ 424 Executes the split backup action. 425 426 @param configPath: Path to configuration file on disk. 427 @type configPath: String representing a path on disk. 428 429 @param options: Program command-line options. 430 @type options: Options object. 431 432 @param config: Program configuration. 433 @type config: Config object. 434 435 @raise ValueError: Under many generic error conditions 436 @raise IOError: If there are I/O problems reading or writing files 437 """ 438 logger.debug("Executing split extended action.") 439 if config.options is None or config.stage is None: 440 raise ValueError("Cedar Backup configuration is not properly filled in.") 441 local = LocalConfig(xmlPath=configPath) 442 dailyDirs = findDailyDirs(config.stage.targetDir, SPLIT_INDICATOR) 443 for dailyDir in dailyDirs: 444 _splitDailyDir(dailyDir, local.split.sizeLimit, local.split.splitSize, 445 config.options.backupUser, config.options.backupGroup) 446 writeIndicatorFile(dailyDir, SPLIT_INDICATOR, config.options.backupUser, config.options.backupGroup) 447 logger.info("Executed the split extended action successfully.")
448
449 450 ############################## 451 # _splitDailyDir() function 452 ############################## 453 454 -def _splitDailyDir(dailyDir, sizeLimit, splitSize, backupUser, backupGroup):
455 """ 456 Splits large files in a daily staging directory. 457 458 Files that match INDICATOR_PATTERNS (i.e. C{"cback.store"}, 459 C{"cback.stage"}, etc.) are assumed to be indicator files and are ignored. 460 All other files are split. 461 462 @param dailyDir: Daily directory to encrypt 463 @param sizeLimit: Size limit, in bytes 464 @param splitSize: Split size, in bytes 465 @param backupUser: User that target files should be owned by 466 @param backupGroup: Group that target files should be owned by 467 468 @raise ValueError: If the encrypt mode is not supported. 469 @raise ValueError: If the daily staging directory does not exist. 470 """ 471 logger.debug("Begin splitting contents of [%s].", dailyDir) 472 fileList = getBackupFiles(dailyDir) # ignores indicator files 473 for path in fileList: 474 size = float(os.stat(path).st_size) 475 if size > sizeLimit: 476 _splitFile(path, splitSize, backupUser, backupGroup, removeSource=True) 477 logger.debug("Completed splitting contents of [%s].", dailyDir)
478
479 480 ######################## 481 # _splitFile() function 482 ######################## 483 484 -def _splitFile(sourcePath, splitSize, backupUser, backupGroup, removeSource=False):
485 """ 486 Splits the source file into chunks of the indicated size. 487 488 The split files will be owned by the indicated backup user and group. If 489 C{removeSource} is C{True}, then the source file will be removed after it is 490 successfully split. 491 492 @param sourcePath: Absolute path of the source file to split 493 @param splitSize: Encryption mode (only "gpg" is allowed) 494 @param backupUser: User that target files should be owned by 495 @param backupGroup: Group that target files should be owned by 496 @param removeSource: Indicates whether to remove the source file 497 498 @raise IOError: If there is a problem accessing, splitting or removing the source file. 499 """ 500 cwd = os.getcwd() 501 try: 502 if not os.path.exists(sourcePath): 503 raise ValueError("Source path [%s] does not exist." % sourcePath) 504 dirname = os.path.dirname(sourcePath) 505 filename = os.path.basename(sourcePath) 506 prefix = "%s_" % filename 507 bytes = int(splitSize.bytes) # pylint: disable=W0622 508 os.chdir(dirname) # need to operate from directory that we want files written to 509 command = resolveCommand(SPLIT_COMMAND) 510 args = [ "--verbose", "--numeric-suffixes", "--suffix-length=5", "--bytes=%d" % bytes, filename, prefix, ] 511 (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=False) 512 if result != 0: 513 raise IOError("Error [%d] calling split for [%s]." % (result, sourcePath)) 514 pattern = re.compile(r"(creating file [`'])(%s)(.*)(')" % prefix) 515 match = pattern.search(output[-1:][0]) 516 if match is None: 517 raise IOError("Unable to parse output from split command.") 518 value = int(match.group(3).strip()) 519 for index in range(0, value): 520 path = "%s%05d" % (prefix, index) 521 if not os.path.exists(path): 522 raise IOError("After call to split, expected file [%s] does not exist." % path) 523 changeOwnership(path, backupUser, backupGroup) 524 if removeSource: 525 if os.path.exists(sourcePath): 526 try: 527 os.remove(sourcePath) 528 logger.debug("Completed removing old file [%s].", sourcePath) 529 except: 530 raise IOError("Failed to remove file [%s] after splitting it." % (sourcePath)) 531 finally: 532 os.chdir(cwd)
533