Package CedarBackup2 :: Package extend :: Module amazons3
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.extend.amazons3

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2014-2015 Kenneth J. Pronovici. 
 12  # All rights reserved. 
 13  # 
 14  # This program is free software; you can redistribute it and/or 
 15  # modify it under the terms of the GNU General Public License, 
 16  # Version 2, as published by the Free Software Foundation. 
 17  # 
 18  # This program is distributed in the hope that it will be useful, 
 19  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 21  # 
 22  # Copies of the GNU General Public License are available from 
 23  # the Free Software Foundation website, http://www.gnu.org/. 
 24  # 
 25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 26  # 
 27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 28  # Language : Python (>= 2.5) 
 29  # Project  : Official Cedar Backup Extensions 
 30  # Revision : $Id: amazons3.py 1097 2015-01-05 20:43:36Z pronovic $ 
 31  # Purpose  : "Store" type extension that writes data to Amazon S3. 
 32  # 
 33  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 34   
 35  ######################################################################## 
 36  # Module documentation 
 37  ######################################################################## 
 38   
 39  """ 
 40  Store-type extension that writes data to Amazon S3. 
 41   
 42  This extension requires a new configuration section <amazons3> and is intended 
 43  to be run immediately after the standard stage action, replacing the standard 
 44  store action.  Aside from its own configuration, it requires the options and 
 45  staging configuration sections in the standard Cedar Backup configuration file. 
 46  Since it is intended to replace the store action, it does not rely on any store 
 47  configuration. 
 48   
 49  The underlying functionality relies on the U{AWS CLI interface 
 50  <http://aws.amazon.com/documentation/cli/>}.  Before you use this extension, 
 51  you need to set up your Amazon S3 account and configure the AWS CLI connection 
 52  per Amazon's documentation.  The extension assumes that the backup is being 
 53  executed as root, and switches over to the configured backup user to 
 54  communicate with AWS.  So, make sure you configure AWS CLI as the backup user 
 55  and not root. 
 56   
 57  You can optionally configure Cedar Backup to encrypt data before sending it 
 58  to S3.  To do that, provide a complete command line using the C{${input}} and 
 59  C{${output}} variables to represent the original input file and the encrypted 
 60  output file.  This command will be executed as the backup user.   
 61   
 62  For instance, you can use something like this with GPG:: 
 63   
 64     /usr/bin/gpg -c --no-use-agent --batch --yes --passphrase-file /home/backup/.passphrase -o ${output} ${input} 
 65   
 66  The GPG mechanism depends on a strong passphrase for security.  One way to 
 67  generate a strong passphrase is using your system random number generator, i.e.:: 
 68   
 69     dd if=/dev/urandom count=20 bs=1 | xxd -ps 
 70   
 71  (See U{StackExchange <http://security.stackexchange.com/questions/14867/gpg-encryption-security>} 
 72  for more details about that advice.) If you decide to use encryption, make sure 
 73  you save off the passphrase in a safe place, so you can get at your backup data 
 74  later if you need to.  And obviously, make sure to set permissions on the 
 75  passphrase file so it can only be read by the backup user. 
 76   
 77  This extension was written for and tested on Linux.  It will throw an exception 
 78  if run on Windows. 
 79   
 80  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 81  """ 
 82   
 83  ######################################################################## 
 84  # Imported modules 
 85  ######################################################################## 
 86   
 87  # System modules 
 88  import sys 
 89  import os 
 90  import logging 
 91  import tempfile 
 92  import datetime 
 93  import json 
 94  import shutil 
 95   
 96  # Cedar Backup modules 
 97  from CedarBackup2.filesystem import FilesystemList, BackupFileList 
 98  from CedarBackup2.util import resolveCommand, executeCommand, isRunningAsRoot, changeOwnership, isStartOfWeek 
 99  from CedarBackup2.xmlutil import createInputDom, addContainerNode, addBooleanNode, addStringNode, addLongNode 
100  from CedarBackup2.xmlutil import readFirstChild, readString, readBoolean, readLong 
101  from CedarBackup2.actions.util import writeIndicatorFile 
102  from CedarBackup2.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR 
103   
104   
105  ######################################################################## 
106  # Module-wide constants and variables 
107  ######################################################################## 
108   
109  logger = logging.getLogger("CedarBackup2.log.extend.amazons3") 
110   
111  SU_COMMAND    = [ "su" ] 
112  AWS_COMMAND   = [ "aws" ] 
113   
114  STORE_INDICATOR = "cback.amazons3" 
115 116 117 ######################################################################## 118 # AmazonS3Config class definition 119 ######################################################################## 120 121 -class AmazonS3Config(object):
122 123 """ 124 Class representing Amazon S3 configuration. 125 126 Amazon S3 configuration is used for storing backup data in Amazon's S3 cloud 127 storage using the C{s3cmd} tool. 128 129 The following restrictions exist on data in this class: 130 131 - The s3Bucket value must be a non-empty string 132 - The encryptCommand value, if set, must be a non-empty string 133 - The full backup size limit, if set, must be a number of bytes >= 0 134 - The incremental backup size limit, if set, must be a number of bytes >= 0 135 136 @sort: __init__, __repr__, __str__, __cmp__, warnMidnite, s3Bucket 137 """ 138
139 - def __init__(self, warnMidnite=None, s3Bucket=None, encryptCommand=None, 140 fullBackupSizeLimit=None, incrementalBackupSizeLimit=None):
141 """ 142 Constructor for the C{AmazonS3Config} class. 143 144 @param warnMidnite: Whether to generate warnings for crossing midnite. 145 @param s3Bucket: Name of the Amazon S3 bucket in which to store the data 146 @param encryptCommand: Command used to encrypt backup data before upload to S3 147 @param fullBackupSizeLimit: Maximum size of a full backup, in bytes 148 @param incrementalBackupSizeLimit: Maximum size of an incremental backup, in bytes 149 150 @raise ValueError: If one of the values is invalid. 151 """ 152 self._warnMidnite = None 153 self._s3Bucket = None 154 self._encryptCommand = None 155 self._fullBackupSizeLimit = None 156 self._incrementalBackupSizeLimit = None 157 self.warnMidnite = warnMidnite 158 self.s3Bucket = s3Bucket 159 self.encryptCommand = encryptCommand 160 self.fullBackupSizeLimit = fullBackupSizeLimit 161 self.incrementalBackupSizeLimit = incrementalBackupSizeLimit
162
163 - def __repr__(self):
164 """ 165 Official string representation for class instance. 166 """ 167 return "AmazonS3Config(%s, %s, %s, %s, %s)" % (self.warnMidnite, self.s3Bucket, self.encryptCommand, 168 self.fullBackupSizeLimit, self.incrementalBackupSizeLimit)
169
170 - def __str__(self):
171 """ 172 Informal string representation for class instance. 173 """ 174 return self.__repr__()
175
176 - def __cmp__(self, other):
177 """ 178 Definition of equals operator for this class. 179 @param other: Other object to compare to. 180 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 181 """ 182 if other is None: 183 return 1 184 if self.warnMidnite != other.warnMidnite: 185 if self.warnMidnite < other.warnMidnite: 186 return -1 187 else: 188 return 1 189 if self.s3Bucket != other.s3Bucket: 190 if self.s3Bucket < other.s3Bucket: 191 return -1 192 else: 193 return 1 194 if self.encryptCommand != other.encryptCommand: 195 if self.encryptCommand < other.encryptCommand: 196 return -1 197 else: 198 return 1 199 if self.fullBackupSizeLimit != other.fullBackupSizeLimit: 200 if self.fullBackupSizeLimit < other.fullBackupSizeLimit: 201 return -1 202 else: 203 return 1 204 if self.incrementalBackupSizeLimit != other.incrementalBackupSizeLimit: 205 if self.incrementalBackupSizeLimit < other.incrementalBackupSizeLimit: 206 return -1 207 else: 208 return 1 209 return 0
210
211 - def _setWarnMidnite(self, value):
212 """ 213 Property target used to set the midnite warning flag. 214 No validations, but we normalize the value to C{True} or C{False}. 215 """ 216 if value: 217 self._warnMidnite = True 218 else: 219 self._warnMidnite = False
220
221 - def _getWarnMidnite(self):
222 """ 223 Property target used to get the midnite warning flag. 224 """ 225 return self._warnMidnite
226
227 - def _setS3Bucket(self, value):
228 """ 229 Property target used to set the S3 bucket. 230 """ 231 if value is not None: 232 if len(value) < 1: 233 raise ValueError("S3 bucket must be non-empty string.") 234 self._s3Bucket = value
235
236 - def _getS3Bucket(self):
237 """ 238 Property target used to get the S3 bucket. 239 """ 240 return self._s3Bucket
241
242 - def _setEncryptCommand(self, value):
243 """ 244 Property target used to set the encrypt command. 245 """ 246 if value is not None: 247 if len(value) < 1: 248 raise ValueError("Encrypt command must be non-empty string.") 249 self._encryptCommand = value
250
251 - def _getEncryptCommand(self):
252 """ 253 Property target used to get the encrypt command. 254 """ 255 return self._encryptCommand
256
257 - def _setFullBackupSizeLimit(self, value):
258 """ 259 Property target used to set the full backup size limit. 260 The value must be an integer >= 0. 261 @raise ValueError: If the value is not valid. 262 """ 263 if value is None: 264 self._fullBackupSizeLimit = None 265 else: 266 try: 267 value = int(value) 268 except TypeError: 269 raise ValueError("Full backup size limit must be an integer >= 0.") 270 if value < 0: 271 raise ValueError("Full backup size limit must be an integer >= 0.") 272 self._fullBackupSizeLimit = value
273
274 - def _getFullBackupSizeLimit(self):
275 """ 276 Property target used to get the full backup size limit. 277 """ 278 return self._fullBackupSizeLimit
279
280 - def _setIncrementalBackupSizeLimit(self, value):
281 """ 282 Property target used to set the incremental backup size limit. 283 The value must be an integer >= 0. 284 @raise ValueError: If the value is not valid. 285 """ 286 if value is None: 287 self._incrementalBackupSizeLimit = None 288 else: 289 try: 290 value = int(value) 291 except TypeError: 292 raise ValueError("Incremental backup size limit must be an integer >= 0.") 293 if value < 0: 294 raise ValueError("Incremental backup size limit must be an integer >= 0.") 295 self._incrementalBackupSizeLimit = value
296
298 """ 299 Property target used to get the incremental backup size limit. 300 """ 301 return self._incrementalBackupSizeLimit
302 303 warnMidnite = property(_getWarnMidnite, _setWarnMidnite, None, "Whether to generate warnings for crossing midnite.") 304 s3Bucket = property(_getS3Bucket, _setS3Bucket, None, doc="Amazon S3 Bucket in which to store data") 305 encryptCommand = property(_getEncryptCommand, _setEncryptCommand, None, doc="Command used to encrypt data before upload to S3") 306 fullBackupSizeLimit = property(_getFullBackupSizeLimit, _setFullBackupSizeLimit, None, 307 doc="Maximum size of a full backup, in bytes") 308 incrementalBackupSizeLimit = property(_getIncrementalBackupSizeLimit, _setIncrementalBackupSizeLimit, None, 309 doc="Maximum size of an incremental backup, in bytes")
310
311 312 ######################################################################## 313 # LocalConfig class definition 314 ######################################################################## 315 316 -class LocalConfig(object):
317 318 """ 319 Class representing this extension's configuration document. 320 321 This is not a general-purpose configuration object like the main Cedar 322 Backup configuration object. Instead, it just knows how to parse and emit 323 amazons3-specific configuration values. Third parties who need to read and 324 write configuration related to this extension should access it through the 325 constructor, C{validate} and C{addConfig} methods. 326 327 @note: Lists within this class are "unordered" for equality comparisons. 328 329 @sort: __init__, __repr__, __str__, __cmp__, amazons3, validate, addConfig 330 """ 331
332 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
333 """ 334 Initializes a configuration object. 335 336 If you initialize the object without passing either C{xmlData} or 337 C{xmlPath} then configuration will be empty and will be invalid until it 338 is filled in properly. 339 340 No reference to the original XML data or original path is saved off by 341 this class. Once the data has been parsed (successfully or not) this 342 original information is discarded. 343 344 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate} 345 method will be called (with its default arguments) against configuration 346 after successfully parsing any passed-in XML. Keep in mind that even if 347 C{validate} is C{False}, it might not be possible to parse the passed-in 348 XML document if lower-level validations fail. 349 350 @note: It is strongly suggested that the C{validate} option always be set 351 to C{True} (the default) unless there is a specific need to read in 352 invalid configuration from disk. 353 354 @param xmlData: XML data representing configuration. 355 @type xmlData: String data. 356 357 @param xmlPath: Path to an XML file on disk. 358 @type xmlPath: Absolute path to a file on disk. 359 360 @param validate: Validate the document after parsing it. 361 @type validate: Boolean true/false. 362 363 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in. 364 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed. 365 @raise ValueError: If the parsed configuration document is not valid. 366 """ 367 self._amazons3 = None 368 self.amazons3 = None 369 if xmlData is not None and xmlPath is not None: 370 raise ValueError("Use either xmlData or xmlPath, but not both.") 371 if xmlData is not None: 372 self._parseXmlData(xmlData) 373 if validate: 374 self.validate() 375 elif xmlPath is not None: 376 xmlData = open(xmlPath).read() 377 self._parseXmlData(xmlData) 378 if validate: 379 self.validate()
380
381 - def __repr__(self):
382 """ 383 Official string representation for class instance. 384 """ 385 return "LocalConfig(%s)" % (self.amazons3)
386
387 - def __str__(self):
388 """ 389 Informal string representation for class instance. 390 """ 391 return self.__repr__()
392
393 - def __cmp__(self, other):
394 """ 395 Definition of equals operator for this class. 396 Lists within this class are "unordered" for equality comparisons. 397 @param other: Other object to compare to. 398 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 399 """ 400 if other is None: 401 return 1 402 if self.amazons3 != other.amazons3: 403 if self.amazons3 < other.amazons3: 404 return -1 405 else: 406 return 1 407 return 0
408
409 - def _setAmazonS3(self, value):
410 """ 411 Property target used to set the amazons3 configuration value. 412 If not C{None}, the value must be a C{AmazonS3Config} object. 413 @raise ValueError: If the value is not a C{AmazonS3Config} 414 """ 415 if value is None: 416 self._amazons3 = None 417 else: 418 if not isinstance(value, AmazonS3Config): 419 raise ValueError("Value must be a C{AmazonS3Config} object.") 420 self._amazons3 = value
421
422 - def _getAmazonS3(self):
423 """ 424 Property target used to get the amazons3 configuration value. 425 """ 426 return self._amazons3
427 428 amazons3 = property(_getAmazonS3, _setAmazonS3, None, "AmazonS3 configuration in terms of a C{AmazonS3Config} object.") 429
430 - def validate(self):
431 """ 432 Validates configuration represented by the object. 433 434 AmazonS3 configuration must be filled in. Within that, the s3Bucket target must be filled in 435 436 @raise ValueError: If one of the validations fails. 437 """ 438 if self.amazons3 is None: 439 raise ValueError("AmazonS3 section is required.") 440 if self.amazons3.s3Bucket is None: 441 raise ValueError("AmazonS3 s3Bucket must be set.")
442
443 - def addConfig(self, xmlDom, parentNode):
444 """ 445 Adds an <amazons3> configuration section as the next child of a parent. 446 447 Third parties should use this function to write configuration related to 448 this extension. 449 450 We add the following fields to the document:: 451 452 warnMidnite //cb_config/amazons3/warn_midnite 453 s3Bucket //cb_config/amazons3/s3_bucket 454 encryptCommand //cb_config/amazons3/encrypt 455 fullBackupSizeLimit //cb_config/amazons3/full_size_limit 456 incrementalBackupSizeLimit //cb_config/amazons3/incr_size_limit 457 458 @param xmlDom: DOM tree as from C{impl.createDocument()}. 459 @param parentNode: Parent that the section should be appended to. 460 """ 461 if self.amazons3 is not None: 462 sectionNode = addContainerNode(xmlDom, parentNode, "amazons3") 463 addBooleanNode(xmlDom, sectionNode, "warn_midnite", self.amazons3.warnMidnite) 464 addStringNode(xmlDom, sectionNode, "s3_bucket", self.amazons3.s3Bucket) 465 addStringNode(xmlDom, sectionNode, "encrypt", self.amazons3.encryptCommand) 466 addLongNode(xmlDom, sectionNode, "full_size_limit", self.amazons3.fullBackupSizeLimit) 467 addLongNode(xmlDom, sectionNode, "incr_size_limit", self.amazons3.incrementalBackupSizeLimit)
468
469 - def _parseXmlData(self, xmlData):
470 """ 471 Internal method to parse an XML string into the object. 472 473 This method parses the XML document into a DOM tree (C{xmlDom}) and then 474 calls a static method to parse the amazons3 configuration section. 475 476 @param xmlData: XML data to be parsed 477 @type xmlData: String data 478 479 @raise ValueError: If the XML cannot be successfully parsed. 480 """ 481 (xmlDom, parentNode) = createInputDom(xmlData) 482 self._amazons3 = LocalConfig._parseAmazonS3(parentNode)
483 484 @staticmethod
485 - def _parseAmazonS3(parent):
486 """ 487 Parses an amazons3 configuration section. 488 489 We read the following individual fields:: 490 491 warnMidnite //cb_config/amazons3/warn_midnite 492 s3Bucket //cb_config/amazons3/s3_bucket 493 encryptCommand //cb_config/amazons3/encrypt 494 fullBackupSizeLimit //cb_config/amazons3/full_size_limit 495 incrementalBackupSizeLimit //cb_config/amazons3/incr_size_limit 496 497 @param parent: Parent node to search beneath. 498 499 @return: C{AmazonS3Config} object or C{None} if the section does not exist. 500 @raise ValueError: If some filled-in value is invalid. 501 """ 502 amazons3 = None 503 section = readFirstChild(parent, "amazons3") 504 if section is not None: 505 amazons3 = AmazonS3Config() 506 amazons3.warnMidnite = readBoolean(section, "warn_midnite") 507 amazons3.s3Bucket = readString(section, "s3_bucket") 508 amazons3.encryptCommand = readString(section, "encrypt") 509 amazons3.fullBackupSizeLimit = readLong(section, "full_size_limit") 510 amazons3.incrementalBackupSizeLimit = readLong(section, "incr_size_limit") 511 return amazons3
512
513 514 ######################################################################## 515 # Public functions 516 ######################################################################## 517 518 ########################### 519 # executeAction() function 520 ########################### 521 522 -def executeAction(configPath, options, config):
523 """ 524 Executes the amazons3 backup action. 525 526 @param configPath: Path to configuration file on disk. 527 @type configPath: String representing a path on disk. 528 529 @param options: Program command-line options. 530 @type options: Options object. 531 532 @param config: Program configuration. 533 @type config: Config object. 534 535 @raise ValueError: Under many generic error conditions 536 @raise IOError: If there are I/O problems reading or writing files 537 """ 538 logger.debug("Executing amazons3 extended action.") 539 if not isRunningAsRoot(): 540 logger.error("Error: the amazons3 extended action must be run as root.") 541 raise ValueError("The amazons3 extended action must be run as root.") 542 if sys.platform == "win32": 543 logger.error("Error: the amazons3 extended action is not supported on Windows.") 544 raise ValueError("The amazons3 extended action is not supported on Windows.") 545 if config.options is None or config.stage is None: 546 raise ValueError("Cedar Backup configuration is not properly filled in.") 547 local = LocalConfig(xmlPath=configPath) 548 stagingDirs = _findCorrectDailyDir(options, config, local) 549 _applySizeLimits(options, config, local, stagingDirs) 550 _writeToAmazonS3(config, local, stagingDirs) 551 _writeStoreIndicator(config, stagingDirs) 552 logger.info("Executed the amazons3 extended action successfully.")
553
554 555 ######################################################################## 556 # Private utility functions 557 ######################################################################## 558 559 ######################### 560 # _findCorrectDailyDir() 561 ######################### 562 563 -def _findCorrectDailyDir(options, config, local):
564 """ 565 Finds the correct daily staging directory to be written to Amazon S3. 566 567 This is substantially similar to the same function in store.py. The 568 main difference is that it doesn't rely on store configuration at all. 569 570 @param options: Options object. 571 @param config: Config object. 572 @param local: Local config object. 573 574 @return: Correct staging dir, as a dict mapping directory to date suffix. 575 @raise IOError: If the staging directory cannot be found. 576 """ 577 oneDay = datetime.timedelta(days=1) 578 today = datetime.date.today() 579 yesterday = today - oneDay 580 tomorrow = today + oneDay 581 todayDate = today.strftime(DIR_TIME_FORMAT) 582 yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT) 583 tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT) 584 todayPath = os.path.join(config.stage.targetDir, todayDate) 585 yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate) 586 tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate) 587 todayStageInd = os.path.join(todayPath, STAGE_INDICATOR) 588 yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR) 589 tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR) 590 todayStoreInd = os.path.join(todayPath, STORE_INDICATOR) 591 yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR) 592 tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR) 593 if options.full: 594 if os.path.isdir(todayPath) and os.path.exists(todayStageInd): 595 logger.info("Amazon S3 process will use current day's staging directory [%s]" % todayPath) 596 return { todayPath:todayDate } 597 raise IOError("Unable to find staging directory to process (only tried today due to full option).") 598 else: 599 if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd): 600 logger.info("Amazon S3 process will use current day's staging directory [%s]" % todayPath) 601 return { todayPath:todayDate } 602 elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd): 603 logger.info("Amazon S3 process will use previous day's staging directory [%s]" % yesterdayPath) 604 if local.amazons3.warnMidnite: 605 logger.warn("Warning: Amazon S3 process crossed midnite boundary to find data.") 606 return { yesterdayPath:yesterdayDate } 607 elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd): 608 logger.info("Amazon S3 process will use next day's staging directory [%s]" % tomorrowPath) 609 if local.amazons3.warnMidnite: 610 logger.warn("Warning: Amazon S3 process crossed midnite boundary to find data.") 611 return { tomorrowPath:tomorrowDate } 612 raise IOError("Unable to find unused staging directory to process (tried today, yesterday, tomorrow).")
613
614 615 ############################## 616 # _applySizeLimits() function 617 ############################## 618 619 -def _applySizeLimits(options, config, local, stagingDirs):
620 """ 621 Apply size limits, throwing an exception if any limits are exceeded. 622 623 Size limits are optional. If a limit is set to None, it does not apply. 624 The full size limit applies if the full option is set or if today is the 625 start of the week. The incremental size limit applies otherwise. Limits 626 are applied to the total size of all the relevant staging directories. 627 628 @param options: Options object. 629 @param config: Config object. 630 @param local: Local config object. 631 @param stagingDirs: Dictionary mapping directory path to date suffix. 632 633 @raise ValueError: Under many generic error conditions 634 @raise ValueError: If a size limit has been exceeded 635 """ 636 if options.full or isStartOfWeek(config.options.startingDay): 637 logger.debug("Using Amazon S3 size limit for full backups.") 638 limit = local.amazons3.fullBackupSizeLimit 639 else: 640 logger.debug("Using Amazon S3 size limit for incremental backups.") 641 limit = local.amazons3.incrementalBackupSizeLimit 642 if limit is None: 643 logger.debug("No Amazon S3 size limit will be applied.") 644 else: 645 logger.debug("Amazon S3 size limit is: %d bytes" % limit) 646 contents = BackupFileList() 647 for stagingDir in stagingDirs: 648 contents.addDirContents(stagingDir) 649 total = contents.totalSize() 650 logger.debug("Amazon S3 backup size is is: %d bytes" % total) 651 if total > limit: 652 logger.error("Amazon S3 size limit exceeded: %.0f bytes > %d bytes" % (total, limit)) 653 raise ValueError("Amazon S3 size limit exceeded: %.0f bytes > %d bytes" % (total, limit)) 654 else: 655 logger.info("Total size does not exceed Amazon S3 size limit, so backup can continue.")
656
657 658 ############################## 659 # _writeToAmazonS3() function 660 ############################## 661 662 -def _writeToAmazonS3(config, local, stagingDirs):
663 """ 664 Writes the indicated staging directories to an Amazon S3 bucket. 665 666 Each of the staging directories listed in C{stagingDirs} will be written to 667 the configured Amazon S3 bucket from local configuration. The directories 668 will be placed into the image at the root by date, so staging directory 669 C{/opt/stage/2005/02/10} will be placed into the S3 bucket at C{/2005/02/10}. 670 If an encrypt commmand is provided, the files will be encrypted first. 671 672 @param config: Config object. 673 @param local: Local config object. 674 @param stagingDirs: Dictionary mapping directory path to date suffix. 675 676 @raise ValueError: Under many generic error conditions 677 @raise IOError: If there is a problem writing to Amazon S3 678 """ 679 for stagingDir in stagingDirs.keys(): 680 logger.debug("Storing stage directory to Amazon S3 [%s]." % stagingDir) 681 dateSuffix = stagingDirs[stagingDir] 682 s3BucketUrl = "s3://%s/%s" % (local.amazons3.s3Bucket, dateSuffix) 683 logger.debug("S3 bucket URL is [%s]" % s3BucketUrl) 684 _clearExistingBackup(config, s3BucketUrl) 685 if local.amazons3.encryptCommand is None: 686 logger.debug("Encryption is disabled; files will be uploaded in cleartext.") 687 _uploadStagingDir(config, stagingDir, s3BucketUrl) 688 _verifyUpload(config, stagingDir, s3BucketUrl) 689 else: 690 logger.debug("Encryption is enabled; files will be uploaded after being encrypted.") 691 encryptedDir = tempfile.mkdtemp(dir=config.options.workingDir) 692 changeOwnership(encryptedDir, config.options.backupUser, config.options.backupGroup) 693 try: 694 _encryptStagingDir(config, local, stagingDir, encryptedDir) 695 _uploadStagingDir(config, encryptedDir, s3BucketUrl) 696 _verifyUpload(config, encryptedDir, s3BucketUrl) 697 finally: 698 if os.path.exists(encryptedDir): 699 shutil.rmtree(encryptedDir)
700
701 702 ################################## 703 # _writeStoreIndicator() function 704 ################################## 705 706 -def _writeStoreIndicator(config, stagingDirs):
707 """ 708 Writes a store indicator file into staging directories. 709 @param config: Config object. 710 @param stagingDirs: Dictionary mapping directory path to date suffix. 711 """ 712 for stagingDir in stagingDirs.keys(): 713 writeIndicatorFile(stagingDir, STORE_INDICATOR, 714 config.options.backupUser, 715 config.options.backupGroup)
716
717 718 ################################## 719 # _clearExistingBackup() function 720 ################################## 721 722 -def _clearExistingBackup(config, s3BucketUrl):
723 """ 724 Clear any existing backup files for an S3 bucket URL. 725 @param config: Config object. 726 @param s3BucketUrl: S3 bucket URL associated with the staging directory 727 """ 728 suCommand = resolveCommand(SU_COMMAND) 729 awsCommand = resolveCommand(AWS_COMMAND) 730 actualCommand = "%s s3 rm --recursive %s/" % (awsCommand[0], s3BucketUrl) 731 result = executeCommand(suCommand, [config.options.backupUser, "-c", actualCommand])[0] 732 if result != 0: 733 raise IOError("Error [%d] calling AWS CLI to clear existing backup for [%s]." % (result, s3BucketUrl)) 734 logger.debug("Completed clearing any existing backup in S3 for [%s]" % s3BucketUrl)
735
736 737 ############################### 738 # _uploadStagingDir() function 739 ############################### 740 741 -def _uploadStagingDir(config, stagingDir, s3BucketUrl):
742 """ 743 Upload the contents of a staging directory out to the Amazon S3 cloud. 744 @param config: Config object. 745 @param stagingDir: Staging directory to upload 746 @param s3BucketUrl: S3 bucket URL associated with the staging directory 747 """ 748 suCommand = resolveCommand(SU_COMMAND) 749 awsCommand = resolveCommand(AWS_COMMAND) 750 actualCommand = "%s s3 cp --recursive %s/ %s/" % (awsCommand[0], stagingDir, s3BucketUrl) 751 result = executeCommand(suCommand, [config.options.backupUser, "-c", actualCommand])[0] 752 if result != 0: 753 raise IOError("Error [%d] calling AWS CLI to upload staging directory to [%s]." % (result, s3BucketUrl)) 754 logger.debug("Completed uploading staging dir [%s] to [%s]" % (stagingDir, s3BucketUrl))
755
756 757 ########################### 758 # _verifyUpload() function 759 ########################### 760 761 -def _verifyUpload(config, stagingDir, s3BucketUrl):
762 """ 763 Verify that a staging directory was properly uploaded to the Amazon S3 cloud. 764 @param config: Config object. 765 @param stagingDir: Staging directory to verify 766 @param s3BucketUrl: S3 bucket URL associated with the staging directory 767 """ 768 (bucket, prefix) = s3BucketUrl.replace("s3://", "").split("/", 1) 769 suCommand = resolveCommand(SU_COMMAND) 770 awsCommand = resolveCommand(AWS_COMMAND) 771 query = "Contents[].{Key: Key, Size: Size}" 772 actualCommand = "%s s3api list-objects --bucket %s --prefix %s --query '%s'" % (awsCommand[0], bucket, prefix, query) 773 (result, data) = executeCommand(suCommand, [config.options.backupUser, "-c", actualCommand], returnOutput=True) 774 if result != 0: 775 raise IOError("Error [%d] calling AWS CLI verify upload to [%s]." % (result, s3BucketUrl)) 776 contents = { } 777 for entry in json.loads("".join(data)): 778 key = entry["Key"].replace(prefix, "") 779 size = long(entry["Size"]) 780 contents[key] = size 781 files = FilesystemList() 782 files.addDirContents(stagingDir) 783 for entry in files: 784 if os.path.isfile(entry): 785 key = entry.replace(stagingDir, "") 786 size = long(os.stat(entry).st_size) 787 if not key in contents: 788 raise IOError("File was apparently not uploaded: [%s]" % entry) 789 else: 790 if size != contents[key]: 791 raise IOError("File size differs [%s], expected %s bytes but got %s bytes" % (entry, size, contents[key])) 792 logger.debug("Completed verifying upload from [%s] to [%s]." % (stagingDir, s3BucketUrl))
793
794 795 ################################ 796 # _encryptStagingDir() function 797 ################################ 798 799 -def _encryptStagingDir(config, local, stagingDir, encryptedDir):
800 """ 801 Encrypt a staging directory, creating a new directory in the process. 802 @param config: Config object. 803 @param stagingDir: Staging directory to use as source 804 @param encryptedDir: Target directory into which encrypted files should be written 805 """ 806 suCommand = resolveCommand(SU_COMMAND) 807 files = FilesystemList() 808 files.addDirContents(stagingDir) 809 for cleartext in files: 810 if os.path.isfile(cleartext): 811 encrypted = "%s%s" % (encryptedDir, cleartext.replace(stagingDir, "")) 812 if long(os.stat(cleartext).st_size) == 0: 813 open(encrypted, 'a').close() # don't bother encrypting empty files 814 else: 815 actualCommand = local.amazons3.encryptCommand.replace("${input}", cleartext).replace("${output}", encrypted) 816 subdir = os.path.dirname(encrypted) 817 if not os.path.isdir(subdir): 818 os.makedirs(subdir) 819 changeOwnership(subdir, config.options.backupUser, config.options.backupGroup) 820 result = executeCommand(suCommand, [config.options.backupUser, "-c", actualCommand])[0] 821 if result != 0: 822 raise IOError("Error [%d] encrypting [%s]." % (result, cleartext)) 823 logger.debug("Completed encrypting staging directory [%s] into [%s]" % (stagingDir, encryptedDir))
824