Package CedarBackup3 :: Module util
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup3.util

   1  # -*- coding: iso-8859-1 -*- 
   2  # vim: set ft=python ts=3 sw=3 expandtab: 
   3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
   4  # 
   5  #              C E D A R 
   6  #          S O L U T I O N S       "Software done right." 
   7  #           S O F T W A R E 
   8  # 
   9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  10  # 
  11  # Copyright (c) 2004-2008,2010,2015 Kenneth J. Pronovici. 
  12  # All rights reserved. 
  13  # 
  14  # Portions copyright (c) 2001, 2002 Python Software Foundation. 
  15  # All Rights Reserved. 
  16  # 
  17  # This program is free software; you can redistribute it and/or 
  18  # modify it under the terms of the GNU General Public License, 
  19  # Version 2, as published by the Free Software Foundation. 
  20  # 
  21  # This program is distributed in the hope that it will be useful, 
  22  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
  23  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
  24  # 
  25  # Copies of the GNU General Public License are available from 
  26  # the Free Software Foundation website, http://www.gnu.org/. 
  27  # 
  28  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  29  # 
  30  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
  31  # Language : Python 3 (>= 3.4) 
  32  # Project  : Cedar Backup, release 3 
  33  # Purpose  : Provides general-purpose utilities. 
  34  # 
  35  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  36   
  37  ######################################################################## 
  38  # Module documentation 
  39  ######################################################################## 
  40   
  41  """ 
  42  Provides general-purpose utilities. 
  43   
  44  @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList, 
  45         RegexList, _Vertex, DirectedGraph, PathResolverSingleton, 
  46         sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine, 
  47         resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice, 
  48         deriveDayOfWeek, isStartOfWeek, buildNormalizedPath, 
  49         ISO_SECTOR_SIZE, BYTES_PER_SECTOR, 
  50         BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE, 
  51         SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY, 
  52         UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS 
  53   
  54  @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes. 
  55  @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector. 
  56  @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB). 
  57  @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB). 
  58  @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB). 
  59  @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB). 
  60  @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB). 
  61  @var SECONDS_PER_MINUTE: Number of seconds per minute. 
  62  @var MINUTES_PER_HOUR: Number of minutes per hour. 
  63  @var HOURS_PER_DAY: Number of hours per day. 
  64  @var SECONDS_PER_DAY: Number of seconds per day. 
  65  @var UNIT_BYTES: Constant representing the byte (B) unit for conversion. 
  66  @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion. 
  67  @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion. 
  68  @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion. 
  69  @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion. 
  70   
  71  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  72  """ 
  73   
  74   
  75  ######################################################################## 
  76  # Imported modules 
  77  ######################################################################## 
  78   
  79  import sys 
  80  import math 
  81  import os 
  82  import re 
  83  import time 
  84  import logging 
  85  from subprocess import Popen, STDOUT, PIPE 
  86  from functools import total_ordering 
  87  from numbers import Real 
  88  from decimal import Decimal 
  89   
  90  from CedarBackup3.release import VERSION, DATE 
  91  import collections 
  92   
  93  try: 
  94     import pwd 
  95     import grp 
  96     _UID_GID_AVAILABLE = True 
  97  except ImportError: 
  98     _UID_GID_AVAILABLE = False 
  99   
 100   
 101  ######################################################################## 
 102  # Module-wide constants and variables 
 103  ######################################################################## 
 104   
 105  logger = logging.getLogger("CedarBackup3.log.util") 
 106  outputLogger = logging.getLogger("CedarBackup3.output") 
 107   
 108  ISO_SECTOR_SIZE    = 2048.0   # in bytes 
 109  BYTES_PER_SECTOR   = ISO_SECTOR_SIZE 
 110   
 111  BYTES_PER_KBYTE    = 1024.0 
 112  KBYTES_PER_MBYTE   = 1024.0 
 113  MBYTES_PER_GBYTE   = 1024.0 
 114  BYTES_PER_MBYTE    = BYTES_PER_KBYTE * KBYTES_PER_MBYTE 
 115  BYTES_PER_GBYTE    = BYTES_PER_MBYTE * MBYTES_PER_GBYTE 
 116   
 117  SECONDS_PER_MINUTE = 60.0 
 118  MINUTES_PER_HOUR   = 60.0 
 119  HOURS_PER_DAY      = 24.0 
 120  SECONDS_PER_DAY    = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY 
 121   
 122  UNIT_BYTES         = 0 
 123  UNIT_KBYTES        = 1 
 124  UNIT_MBYTES        = 2 
 125  UNIT_GBYTES        = 4 
 126  UNIT_SECTORS       = 3 
 127   
 128  MTAB_FILE          = "/etc/mtab" 
 129   
 130  MOUNT_COMMAND      = [ "mount", ] 
 131  UMOUNT_COMMAND     = [ "umount", ] 
 132   
 133  DEFAULT_LANGUAGE   = "C" 
 134  LANG_VAR           = "LANG" 
 135  LOCALE_VARS        = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE", 
 136                         "LC_CTYPE", "LC_IDENTIFICATION", 
 137                         "LC_MEASUREMENT", "LC_MESSAGES", 
 138                         "LC_MONETARY", "LC_NAME", "LC_NUMERIC", 
 139                         "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ] 
140 141 142 ######################################################################## 143 # UnorderedList class definition 144 ######################################################################## 145 146 -class UnorderedList(list):
147 148 """ 149 Class representing an "unordered list". 150 151 An "unordered list" is a list in which only the contents matter, not the 152 order in which the contents appear in the list. 153 154 For instance, we might be keeping track of set of paths in a list, because 155 it's convenient to have them in that form. However, for comparison 156 purposes, we would only care that the lists contain exactly the same 157 contents, regardless of order. 158 159 I have come up with two reasonable ways of doing this, plus a couple more 160 that would work but would be a pain to implement. My first method is to 161 copy and sort each list, comparing the sorted versions. This will only work 162 if two lists with exactly the same members are guaranteed to sort in exactly 163 the same order. The second way would be to create two Sets and then compare 164 the sets. However, this would lose information about any duplicates in 165 either list. I've decided to go with option #1 for now. I'll modify this 166 code if I run into problems in the future. 167 168 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__}, 169 C{__le__} and C{__lt__} list methods to change the definition of the various 170 comparison operators. In all cases, the comparison is changed to return the 171 result of the original operation I{but instead comparing sorted lists}. 172 This is going to be quite a bit slower than a normal list, so you probably 173 only want to use it on small lists. 174 """ 175
176 - def __eq__(self, other):
177 """ 178 Definition of C{==} operator for this class. 179 @param other: Other object to compare to. 180 @return: True/false depending on whether C{self == other}. 181 """ 182 if other is None: 183 return False 184 selfSorted = UnorderedList.mixedsort(self[:]) 185 otherSorted = UnorderedList.mixedsort(other[:]) 186 return selfSorted.__eq__(otherSorted)
187
188 - def __ne__(self, other):
189 """ 190 Definition of C{!=} operator for this class. 191 @param other: Other object to compare to. 192 @return: True/false depending on whether C{self != other}. 193 """ 194 if other is None: 195 return True 196 selfSorted = UnorderedList.mixedsort(self[:]) 197 otherSorted = UnorderedList.mixedsort(other[:]) 198 return selfSorted.__ne__(otherSorted)
199
200 - def __ge__(self, other):
201 """ 202 Definition of S{>=} operator for this class. 203 @param other: Other object to compare to. 204 @return: True/false depending on whether C{self >= other}. 205 """ 206 if other is None: 207 return True 208 selfSorted = UnorderedList.mixedsort(self[:]) 209 otherSorted = UnorderedList.mixedsort(other[:]) 210 return selfSorted.__ge__(otherSorted)
211
212 - def __gt__(self, other):
213 """ 214 Definition of C{>} operator for this class. 215 @param other: Other object to compare to. 216 @return: True/false depending on whether C{self > other}. 217 """ 218 if other is None: 219 return True 220 selfSorted = UnorderedList.mixedsort(self[:]) 221 otherSorted = UnorderedList.mixedsort(other[:]) 222 return selfSorted.__gt__(otherSorted)
223
224 - def __le__(self, other):
225 """ 226 Definition of S{<=} operator for this class. 227 @param other: Other object to compare to. 228 @return: True/false depending on whether C{self <= other}. 229 """ 230 if other is None: 231 return False 232 selfSorted = UnorderedList.mixedsort(self[:]) 233 otherSorted = UnorderedList.mixedsort(other[:]) 234 return selfSorted.__le__(otherSorted)
235
236 - def __lt__(self, other):
237 """ 238 Definition of C{<} operator for this class. 239 @param other: Other object to compare to. 240 @return: True/false depending on whether C{self < other}. 241 """ 242 if other is None: 243 return False 244 selfSorted = UnorderedList.mixedsort(self[:]) 245 otherSorted = UnorderedList.mixedsort(other[:]) 246 return selfSorted.__lt__(otherSorted)
247 248 @staticmethod
249 - def mixedsort(value):
250 """ 251 Sort a list, making sure we don't blow up if the list happens to include mixed values. 252 @see: http://stackoverflow.com/questions/26575183/how-can-i-get-2-x-like-sorting-behaviour-in-python-3-x 253 """ 254 return sorted(value, key=UnorderedList.mixedkey)
255 256 @staticmethod
257 - def mixedkey(value):
258 """Provide a key for use by mixedsort()""" 259 numeric = Real, Decimal 260 if isinstance(value, numeric): 261 typeinfo = numeric 262 else: 263 typeinfo = type(value) 264 try: 265 x = value < value 266 except TypeError: 267 value = repr(value) 268 return repr(typeinfo), value
269
270 271 ######################################################################## 272 # AbsolutePathList class definition 273 ######################################################################## 274 275 -class AbsolutePathList(UnorderedList):
276 277 """ 278 Class representing a list of absolute paths. 279 280 This is an unordered list. 281 282 We override the C{append}, C{insert} and C{extend} methods to ensure that 283 any item added to the list is an absolute path. 284 285 Each item added to the list is encoded using L{encodePath}. If we don't do 286 this, we have problems trying certain operations between strings and unicode 287 objects, particularly for "odd" filenames that can't be encoded in standard 288 ASCII. 289 """ 290
291 - def append(self, item):
292 """ 293 Overrides the standard C{append} method. 294 @raise ValueError: If item is not an absolute path. 295 """ 296 if not os.path.isabs(item): 297 raise ValueError("Not an absolute path: [%s]" % item) 298 list.append(self, encodePath(item))
299
300 - def insert(self, index, item):
301 """ 302 Overrides the standard C{insert} method. 303 @raise ValueError: If item is not an absolute path. 304 """ 305 if not os.path.isabs(item): 306 raise ValueError("Not an absolute path: [%s]" % item) 307 list.insert(self, index, encodePath(item))
308
309 - def extend(self, seq):
310 """ 311 Overrides the standard C{insert} method. 312 @raise ValueError: If any item is not an absolute path. 313 """ 314 for item in seq: 315 if not os.path.isabs(item): 316 raise ValueError("Not an absolute path: [%s]" % item) 317 for item in seq: 318 list.append(self, encodePath(item))
319
320 321 ######################################################################## 322 # ObjectTypeList class definition 323 ######################################################################## 324 325 -class ObjectTypeList(UnorderedList):
326 327 """ 328 Class representing a list containing only objects with a certain type. 329 330 This is an unordered list. 331 332 We override the C{append}, C{insert} and C{extend} methods to ensure that 333 any item added to the list matches the type that is requested. The 334 comparison uses the built-in C{isinstance}, which should allow subclasses of 335 of the requested type to be added to the list as well. 336 337 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a 338 CollectDir object."} if C{objectName} is C{"CollectDir"}. 339 """ 340
341 - def __init__(self, objectType, objectName):
342 """ 343 Initializes a typed list for a particular type. 344 @param objectType: Type that the list elements must match. 345 @param objectName: Short string containing the "name" of the type. 346 """ 347 super(ObjectTypeList, self).__init__() 348 self.objectType = objectType 349 self.objectName = objectName
350
351 - def append(self, item):
352 """ 353 Overrides the standard C{append} method. 354 @raise ValueError: If item does not match requested type. 355 """ 356 if not isinstance(item, self.objectType): 357 raise ValueError("Item must be a %s object." % self.objectName) 358 list.append(self, item)
359
360 - def insert(self, index, item):
361 """ 362 Overrides the standard C{insert} method. 363 @raise ValueError: If item does not match requested type. 364 """ 365 if not isinstance(item, self.objectType): 366 raise ValueError("Item must be a %s object." % self.objectName) 367 list.insert(self, index, item)
368
369 - def extend(self, seq):
370 """ 371 Overrides the standard C{insert} method. 372 @raise ValueError: If item does not match requested type. 373 """ 374 for item in seq: 375 if not isinstance(item, self.objectType): 376 raise ValueError("All items must be %s objects." % self.objectName) 377 list.extend(self, seq)
378
379 380 ######################################################################## 381 # RestrictedContentList class definition 382 ######################################################################## 383 384 -class RestrictedContentList(UnorderedList):
385 386 """ 387 Class representing a list containing only object with certain values. 388 389 This is an unordered list. 390 391 We override the C{append}, C{insert} and C{extend} methods to ensure that 392 any item added to the list is among the valid values. We use a standard 393 comparison, so pretty much anything can be in the list of valid values. 394 395 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be 396 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}. 397 398 @note: This class doesn't make any attempt to trap for nonsensical 399 arguments. All of the values in the values list should be of the same type 400 (i.e. strings). Then, all list operations also need to be of that type 401 (i.e. you should always insert or append just strings). If you mix types -- 402 for instance lists and strings -- you will likely see AttributeError 403 exceptions or other problems. 404 """ 405
406 - def __init__(self, valuesList, valuesDescr, prefix=None):
407 """ 408 Initializes a list restricted to containing certain values. 409 @param valuesList: List of valid values. 410 @param valuesDescr: Short string describing list of values. 411 @param prefix: Prefix to use in error messages (None results in prefix "Item") 412 """ 413 super(RestrictedContentList, self).__init__() 414 self.prefix = "Item" 415 if prefix is not None: self.prefix = prefix 416 self.valuesList = valuesList 417 self.valuesDescr = valuesDescr
418
419 - def append(self, item):
420 """ 421 Overrides the standard C{append} method. 422 @raise ValueError: If item is not in the values list. 423 """ 424 if item not in self.valuesList: 425 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 426 list.append(self, item)
427
428 - def insert(self, index, item):
429 """ 430 Overrides the standard C{insert} method. 431 @raise ValueError: If item is not in the values list. 432 """ 433 if item not in self.valuesList: 434 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 435 list.insert(self, index, item)
436
437 - def extend(self, seq):
438 """ 439 Overrides the standard C{insert} method. 440 @raise ValueError: If item is not in the values list. 441 """ 442 for item in seq: 443 if item not in self.valuesList: 444 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 445 list.extend(self, seq)
446
447 448 ######################################################################## 449 # RegexMatchList class definition 450 ######################################################################## 451 452 -class RegexMatchList(UnorderedList):
453 454 """ 455 Class representing a list containing only strings that match a regular expression. 456 457 If C{emptyAllowed} is passed in as C{False}, then empty strings are 458 explicitly disallowed, even if they happen to match the regular expression. 459 (C{None} values are always disallowed, since string operations are not 460 permitted on C{None}.) 461 462 This is an unordered list. 463 464 We override the C{append}, C{insert} and C{extend} methods to ensure that 465 any item added to the list matches the indicated regular expression. 466 467 @note: If you try to put values that are not strings into the list, you will 468 likely get either TypeError or AttributeError exceptions as a result. 469 """ 470
471 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
472 """ 473 Initializes a list restricted to containing certain values. 474 @param valuesRegex: Regular expression that must be matched, as a string 475 @param emptyAllowed: Indicates whether empty or None values are allowed. 476 @param prefix: Prefix to use in error messages (None results in prefix "Item") 477 """ 478 super(RegexMatchList, self).__init__() 479 self.prefix = "Item" 480 if prefix is not None: self.prefix = prefix 481 self.valuesRegex = valuesRegex 482 self.emptyAllowed = emptyAllowed 483 self.pattern = re.compile(self.valuesRegex)
484
485 - def append(self, item):
486 """ 487 Overrides the standard C{append} method. 488 @raise ValueError: If item is None 489 @raise ValueError: If item is empty and empty values are not allowed 490 @raise ValueError: If item does not match the configured regular expression 491 """ 492 if item is None or (not self.emptyAllowed and item == ""): 493 raise ValueError("%s cannot be empty." % self.prefix) 494 if not self.pattern.search(item): 495 raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 496 list.append(self, item)
497
498 - def insert(self, index, item):
499 """ 500 Overrides the standard C{insert} method. 501 @raise ValueError: If item is None 502 @raise ValueError: If item is empty and empty values are not allowed 503 @raise ValueError: If item does not match the configured regular expression 504 """ 505 if item is None or (not self.emptyAllowed and item == ""): 506 raise ValueError("%s cannot be empty." % self.prefix) 507 if not self.pattern.search(item): 508 raise ValueError("%s is not valid [%s]" % (self.prefix, item)) 509 list.insert(self, index, item)
510
511 - def extend(self, seq):
512 """ 513 Overrides the standard C{insert} method. 514 @raise ValueError: If any item is None 515 @raise ValueError: If any item is empty and empty values are not allowed 516 @raise ValueError: If any item does not match the configured regular expression 517 """ 518 for item in seq: 519 if item is None or (not self.emptyAllowed and item == ""): 520 raise ValueError("%s cannot be empty." % self.prefix) 521 if not self.pattern.search(item): 522 raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 523 list.extend(self, seq)
524
525 526 ######################################################################## 527 # RegexList class definition 528 ######################################################################## 529 530 -class RegexList(UnorderedList):
531 532 """ 533 Class representing a list of valid regular expression strings. 534 535 This is an unordered list. 536 537 We override the C{append}, C{insert} and C{extend} methods to ensure that 538 any item added to the list is a valid regular expression. 539 """ 540
541 - def append(self, item):
542 """ 543 Overrides the standard C{append} method. 544 @raise ValueError: If item is not an absolute path. 545 """ 546 try: 547 re.compile(item) 548 except re.error: 549 raise ValueError("Not a valid regular expression: [%s]" % item) 550 list.append(self, item)
551
552 - def insert(self, index, item):
553 """ 554 Overrides the standard C{insert} method. 555 @raise ValueError: If item is not an absolute path. 556 """ 557 try: 558 re.compile(item) 559 except re.error: 560 raise ValueError("Not a valid regular expression: [%s]" % item) 561 list.insert(self, index, item)
562
563 - def extend(self, seq):
564 """ 565 Overrides the standard C{insert} method. 566 @raise ValueError: If any item is not an absolute path. 567 """ 568 for item in seq: 569 try: 570 re.compile(item) 571 except re.error: 572 raise ValueError("Not a valid regular expression: [%s]" % item) 573 for item in seq: 574 list.append(self, item)
575
576 577 ######################################################################## 578 # Directed graph implementation 579 ######################################################################## 580 581 -class _Vertex(object):
582 583 """ 584 Represents a vertex (or node) in a directed graph. 585 """ 586
587 - def __init__(self, name):
588 """ 589 Constructor. 590 @param name: Name of this graph vertex. 591 @type name: String value. 592 """ 593 self.name = name 594 self.endpoints = [] 595 self.state = None
596
597 @total_ordering 598 -class DirectedGraph(object):
599 600 """ 601 Represents a directed graph. 602 603 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set 604 B{E} of vertex pairs or edges. In a directed graph, each edge also has an 605 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph} 606 object provides a way to construct a directed graph and execute a depth- 607 first search. 608 609 This data structure was designed based on the graphing chapter in 610 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>}, 611 by Steven S. Skiena. 612 613 This class is intended to be used by Cedar Backup for dependency ordering. 614 Because of this, it's not quite general-purpose. Unlike a "general" graph, 615 every vertex in this graph has at least one edge pointing to it, from a 616 special "start" vertex. This is so no vertices get "lost" either because 617 they have no dependencies or because nothing depends on them. 618 """ 619 620 _UNDISCOVERED = 0 621 _DISCOVERED = 1 622 _EXPLORED = 2 623
624 - def __init__(self, name):
625 """ 626 Directed graph constructor. 627 628 @param name: Name of this graph. 629 @type name: String value. 630 """ 631 if name is None or name == "": 632 raise ValueError("Graph name must be non-empty.") 633 self._name = name 634 self._vertices = {} 635 self._startVertex = _Vertex(None) # start vertex is only vertex with no name
636
637 - def __repr__(self):
638 """ 639 Official string representation for class instance. 640 """ 641 return "DirectedGraph(%s)" % self.name
642
643 - def __str__(self):
644 """ 645 Informal string representation for class instance. 646 """ 647 return self.__repr__()
648
649 - def __eq__(self, other):
650 """Equals operator, implemented in terms of original Python 2 compare operator.""" 651 return self.__cmp__(other) == 0
652
653 - def __lt__(self, other):
654 """Less-than operator, implemented in terms of original Python 2 compare operator.""" 655 return self.__cmp__(other) < 0
656
657 - def __gt__(self, other):
658 """Greater-than operator, implemented in terms of original Python 2 compare operator.""" 659 return self.__cmp__(other) > 0
660
661 - def __cmp__(self, other):
662 """ 663 Original Python 2 comparison operator. 664 @param other: Other object to compare to. 665 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 666 """ 667 # pylint: disable=W0212 668 if other is None: 669 return 1 670 if self.name != other.name: 671 if str(self.name or "") < str(other.name or ""): 672 return -1 673 else: 674 return 1 675 if self._vertices != other._vertices: 676 if self._vertices < other._vertices: 677 return -1 678 else: 679 return 1 680 return 0
681
682 - def _getName(self):
683 """ 684 Property target used to get the graph name. 685 """ 686 return self._name
687 688 name = property(_getName, None, None, "Name of the graph.") 689
690 - def createVertex(self, name):
691 """ 692 Creates a named vertex. 693 @param name: vertex name 694 @raise ValueError: If the vertex name is C{None} or empty. 695 """ 696 if name is None or name == "": 697 raise ValueError("Vertex name must be non-empty.") 698 vertex = _Vertex(name) 699 self._startVertex.endpoints.append(vertex) # so every vertex is connected at least once 700 self._vertices[name] = vertex
701
702 - def createEdge(self, start, finish):
703 """ 704 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex. 705 @param start: Name of start vertex. 706 @param finish: Name of finish vertex. 707 @raise ValueError: If one of the named vertices is unknown. 708 """ 709 try: 710 startVertex = self._vertices[start] 711 finishVertex = self._vertices[finish] 712 startVertex.endpoints.append(finishVertex) 713 except KeyError as e: 714 raise ValueError("Vertex [%s] could not be found." % e)
715
716 - def topologicalSort(self):
717 """ 718 Implements a topological sort of the graph. 719 720 This method also enforces that the graph is a directed acyclic graph, 721 which is a requirement of a topological sort. 722 723 A directed acyclic graph (or "DAG") is a directed graph with no directed 724 cycles. A topological sort of a DAG is an ordering on the vertices such 725 that all edges go from left to right. Only an acyclic graph can have a 726 topological sort, but any DAG has at least one topological sort. 727 728 Since a topological sort only makes sense for an acyclic graph, this 729 method throws an exception if a cycle is found. 730 731 A depth-first search only makes sense if the graph is acyclic. If the 732 graph contains any cycles, it is not possible to determine a consistent 733 ordering for the vertices. 734 735 @note: If a particular vertex has no edges, then its position in the 736 final list depends on the order in which the vertices were created in the 737 graph. If you're using this method to determine a dependency order, this 738 makes sense: a vertex with no dependencies can go anywhere (and will). 739 740 @return: Ordering on the vertices so that all edges go from left to right. 741 742 @raise ValueError: If a cycle is found in the graph. 743 """ 744 ordering = [] 745 for key in self._vertices: 746 vertex = self._vertices[key] 747 vertex.state = self._UNDISCOVERED 748 for key in self._vertices: 749 vertex = self._vertices[key] 750 if vertex.state == self._UNDISCOVERED: 751 self._topologicalSort(self._startVertex, ordering) 752 return ordering
753
754 - def _topologicalSort(self, vertex, ordering):
755 """ 756 Recursive depth first search function implementing topological sort. 757 @param vertex: Vertex to search 758 @param ordering: List of vertices in proper order 759 """ 760 vertex.state = self._DISCOVERED 761 for endpoint in vertex.endpoints: 762 if endpoint.state == self._UNDISCOVERED: 763 self._topologicalSort(endpoint, ordering) 764 elif endpoint.state != self._EXPLORED: 765 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name)) 766 if vertex.name is not None: 767 ordering.insert(0, vertex.name) 768 vertex.state = self._EXPLORED
769
770 771 ######################################################################## 772 # PathResolverSingleton class definition 773 ######################################################################## 774 775 -class PathResolverSingleton(object):
776 777 """ 778 Singleton used for resolving executable paths. 779 780 Various functions throughout Cedar Backup (including extensions) need a way 781 to resolve the path of executables that they use. For instance, the image 782 functionality needs to find the C{mkisofs} executable, and the Subversion 783 extension needs to find the C{svnlook} executable. Cedar Backup's original 784 behavior was to assume that the simple name (C{"svnlook"} or whatever) was 785 available on the caller's C{$PATH}, and to fail otherwise. However, this 786 turns out to be less than ideal, since for instance the root user might not 787 always have executables like C{svnlook} in its path. 788 789 One solution is to specify a path (either via an absolute path or some sort 790 of path insertion or path appending mechanism) that would apply to the 791 C{executeCommand()} function. This is not difficult to implement, but it 792 seem like kind of a "big hammer" solution. Besides that, it might also 793 represent a security flaw (for instance, I prefer not to mess with root's 794 C{$PATH} on the application level if I don't have to). 795 796 The alternative is to set up some sort of configuration for the path to 797 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or 798 whatever. This PathResolverSingleton aims to provide a good solution to the 799 mapping problem. Callers of all sorts (extensions or not) can get an 800 instance of the singleton. Then, they call the C{lookup} method to try and 801 resolve the executable they are looking for. Through the C{lookup} method, 802 the caller can also specify a default to use if a mapping is not found. 803 This way, with no real effort on the part of the caller, behavior can neatly 804 degrade to something equivalent to the current behavior if there is no 805 special mapping or if the singleton was never initialized in the first 806 place. 807 808 Even better, extensions automagically get access to the same resolver 809 functionality, and they don't even need to understand how the mapping 810 happens. All extension authors need to do is document what executables 811 their code requires, and the standard resolver configuration section will 812 meet their needs. 813 814 The class should be initialized once through the constructor somewhere in 815 the main routine. Then, the main routine should call the L{fill} method to 816 fill in the resolver's internal structures. Everyone else who needs to 817 resolve a path will get an instance of the class using L{getInstance} and 818 will then just call the L{lookup} method. 819 820 @cvar _instance: Holds a reference to the singleton 821 @ivar _mapping: Internal mapping from resource name to path. 822 """ 823 824 _instance = None # Holds a reference to singleton instance 825
826 - class _Helper:
827 """Helper class to provide a singleton factory method."""
828 - def __init__(self):
829 pass
830 - def __call__(self, *args, **kw):
831 # pylint: disable=W0212,R0201 832 if PathResolverSingleton._instance is None: 833 obj = PathResolverSingleton() 834 PathResolverSingleton._instance = obj 835 return PathResolverSingleton._instance
836 837 getInstance = _Helper() # Method that callers will use to get an instance 838
839 - def __init__(self, ):
840 """Singleton constructor, which just creates the singleton instance.""" 841 PathResolverSingleton._instance = self 842 self._mapping = { }
843
844 - def lookup(self, name, default=None):
845 """ 846 Looks up name and returns the resolved path associated with the name. 847 @param name: Name of the path resource to resolve. 848 @param default: Default to return if resource cannot be resolved. 849 @return: Resolved path associated with name, or default if name can't be resolved. 850 """ 851 value = default 852 if name in list(self._mapping.keys()): 853 value = self._mapping[name] 854 logger.debug("Resolved command [%s] to [%s].", name, value) 855 return value
856
857 - def fill(self, mapping):
858 """ 859 Fills in the singleton's internal mapping from name to resource. 860 @param mapping: Mapping from resource name to path. 861 @type mapping: Dictionary mapping name to path, both as strings. 862 """ 863 self._mapping = { } 864 for key in list(mapping.keys()): 865 self._mapping[key] = mapping[key]
866
867 868 ######################################################################## 869 # Pipe class definition 870 ######################################################################## 871 872 -class Pipe(Popen):
873 """ 874 Specialized pipe class for use by C{executeCommand}. 875 876 The L{executeCommand} function needs a specialized way of interacting 877 with a pipe. First, C{executeCommand} only reads from the pipe, and 878 never writes to it. Second, C{executeCommand} needs a way to discard all 879 output written to C{stderr}, as a means of simulating the shell 880 C{2>/dev/null} construct. 881 """
882 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
883 stderr = STDOUT 884 if ignoreStderr: 885 devnull = nullDevice() 886 stderr = os.open(devnull, os.O_RDWR) 887 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
888
889 890 ######################################################################## 891 # Diagnostics class definition 892 ######################################################################## 893 894 -class Diagnostics(object):
895 896 """ 897 Class holding runtime diagnostic information. 898 899 Diagnostic information is information that is useful to get from users for 900 debugging purposes. I'm consolidating it all here into one object. 901 902 @sort: __init__, __repr__, __str__ 903 """ 904 # pylint: disable=R0201 905
906 - def __init__(self):
907 """ 908 Constructor for the C{Diagnostics} class. 909 """
910
911 - def __repr__(self):
912 """ 913 Official string representation for class instance. 914 """ 915 return "Diagnostics()"
916
917 - def __str__(self):
918 """ 919 Informal string representation for class instance. 920 """ 921 return self.__repr__()
922
923 - def getValues(self):
924 """ 925 Get a map containing all of the diagnostic values. 926 @return: Map from diagnostic name to diagnostic value. 927 """ 928 values = {} 929 values['version'] = self.version 930 values['interpreter'] = self.interpreter 931 values['platform'] = self.platform 932 values['encoding'] = self.encoding 933 values['locale'] = self.locale 934 values['timestamp'] = self.timestamp 935 return values
936
937 - def printDiagnostics(self, fd=sys.stdout, prefix=""):
938 """ 939 Pretty-print diagnostic information to a file descriptor. 940 @param fd: File descriptor used to print information. 941 @param prefix: Prefix string (if any) to place onto printed lines 942 @note: The C{fd} is used rather than C{print} to facilitate unit testing. 943 """ 944 lines = self._buildDiagnosticLines(prefix) 945 for line in lines: 946 fd.write("%s\n" % line)
947
948 - def logDiagnostics(self, method, prefix=""):
949 """ 950 Pretty-print diagnostic information using a logger method. 951 @param method: Logger method to use for logging (i.e. logger.info) 952 @param prefix: Prefix string (if any) to place onto printed lines 953 """ 954 lines = self._buildDiagnosticLines(prefix) 955 for line in lines: 956 method("%s" % line)
957
958 - def _buildDiagnosticLines(self, prefix=""):
959 """ 960 Build a set of pretty-printed diagnostic lines. 961 @param prefix: Prefix string (if any) to place onto printed lines 962 @return: List of strings, not terminated by newlines. 963 """ 964 values = self.getValues() 965 keys = list(values.keys()) 966 keys.sort() 967 tmax = Diagnostics._getMaxLength(keys) + 3 # three extra dots in output 968 lines = [] 969 for key in keys: 970 title = key.title() 971 title += (tmax - len(title)) * '.' 972 value = values[key] 973 line = "%s%s: %s" % (prefix, title, value) 974 lines.append(line) 975 return lines
976 977 @staticmethod
978 - def _getMaxLength(values):
979 """ 980 Get the maximum length from among a list of strings. 981 """ 982 tmax = 0 983 for value in values: 984 if len(value) > tmax: 985 tmax = len(value) 986 return tmax
987
988 - def _getVersion(self):
989 """ 990 Property target to get the Cedar Backup version. 991 """ 992 return "Cedar Backup %s (%s)" % (VERSION, DATE)
993
994 - def _getInterpreter(self):
995 """ 996 Property target to get the Python interpreter version. 997 """ 998 version = sys.version_info 999 return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3])
1000
1001 - def _getEncoding(self):
1002 """ 1003 Property target to get the filesystem encoding. 1004 """ 1005 return sys.getfilesystemencoding() or sys.getdefaultencoding()
1006
1007 - def _getPlatform(self):
1008 """ 1009 Property target to get the operating system platform. 1010 """ 1011 try: 1012 uname = os.uname() 1013 sysname = uname[0] # i.e. Linux 1014 release = uname[2] # i.e. 2.16.18-2 1015 machine = uname[4] # i.e. i686 1016 return "%s (%s %s %s)" % (sys.platform, sysname, release, machine) 1017 except: 1018 return sys.platform
1019
1020 - def _getLocale(self):
1021 """ 1022 Property target to get the default locale that is in effect. 1023 """ 1024 try: 1025 import locale 1026 return locale.getdefaultlocale()[0] 1027 except: 1028 return "(unknown)"
1029
1030 - def _getTimestamp(self):
1031 """ 1032 Property target to get a current date/time stamp. 1033 """ 1034 try: 1035 import datetime 1036 return datetime.datetime.utcnow().ctime() + " UTC" 1037 except: 1038 return "(unknown)"
1039 1040 version = property(_getVersion, None, None, "Cedar Backup version.") 1041 interpreter = property(_getInterpreter, None, None, "Python interpreter version.") 1042 platform = property(_getPlatform, None, None, "Platform identifying information.") 1043 encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.") 1044 locale = property(_getLocale, None, None, "Locale that is in effect.") 1045 timestamp = property(_getTimestamp, None, None, "Current timestamp.")
1046
1047 1048 ######################################################################## 1049 # General utility functions 1050 ######################################################################## 1051 1052 ###################### 1053 # sortDict() function 1054 ###################### 1055 1056 -def sortDict(d):
1057 """ 1058 Returns the keys of the dictionary sorted by value. 1059 @param d: Dictionary to operate on 1060 @return: List of dictionary keys sorted in order by dictionary value. 1061 """ 1062 items = list(d.items()) 1063 items.sort(key=lambda x: (x[1], x[0])) # sort by value and then by key 1064 return [key for key, value in items]
1065
1066 1067 ######################## 1068 # removeKeys() function 1069 ######################## 1070 1071 -def removeKeys(d, keys):
1072 """ 1073 Removes all of the keys from the dictionary. 1074 The dictionary is altered in-place. 1075 Each key must exist in the dictionary. 1076 @param d: Dictionary to operate on 1077 @param keys: List of keys to remove 1078 @raise KeyError: If one of the keys does not exist 1079 """ 1080 for key in keys: 1081 del d[key]
1082
1083 1084 ######################### 1085 # convertSize() function 1086 ######################### 1087 1088 -def convertSize(size, fromUnit, toUnit):
1089 """ 1090 Converts a size in one unit to a size in another unit. 1091 1092 This is just a convenience function so that the functionality can be 1093 implemented in just one place. Internally, we convert values to bytes and 1094 then to the final unit. 1095 1096 The available units are: 1097 1098 - C{UNIT_BYTES} - Bytes 1099 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B 1100 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB 1101 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB 1102 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B 1103 1104 @param size: Size to convert 1105 @type size: Integer or float value in units of C{fromUnit} 1106 1107 @param fromUnit: Unit to convert from 1108 @type fromUnit: One of the units listed above 1109 1110 @param toUnit: Unit to convert to 1111 @type toUnit: One of the units listed above 1112 1113 @return: Number converted to new unit, as a float. 1114 @raise ValueError: If one of the units is invalid. 1115 """ 1116 if size is None: 1117 raise ValueError("Cannot convert size of None.") 1118 if fromUnit == UNIT_BYTES: 1119 byteSize = float(size) 1120 elif fromUnit == UNIT_KBYTES: 1121 byteSize = float(size) * BYTES_PER_KBYTE 1122 elif fromUnit == UNIT_MBYTES: 1123 byteSize = float(size) * BYTES_PER_MBYTE 1124 elif fromUnit == UNIT_GBYTES: 1125 byteSize = float(size) * BYTES_PER_GBYTE 1126 elif fromUnit == UNIT_SECTORS: 1127 byteSize = float(size) * BYTES_PER_SECTOR 1128 else: 1129 raise ValueError("Unknown 'from' unit %s." % fromUnit) 1130 if toUnit == UNIT_BYTES: 1131 return byteSize 1132 elif toUnit == UNIT_KBYTES: 1133 return byteSize / BYTES_PER_KBYTE 1134 elif toUnit == UNIT_MBYTES: 1135 return byteSize / BYTES_PER_MBYTE 1136 elif toUnit == UNIT_GBYTES: 1137 return byteSize / BYTES_PER_GBYTE 1138 elif toUnit == UNIT_SECTORS: 1139 return byteSize / BYTES_PER_SECTOR 1140 else: 1141 raise ValueError("Unknown 'to' unit %s." % toUnit)
1142
1143 1144 ########################## 1145 # displayBytes() function 1146 ########################## 1147 1148 -def displayBytes(bytes, digits=2): # pylint: disable=W0622
1149 """ 1150 Format a byte quantity so it can be sensibly displayed. 1151 1152 It's rather difficult to look at a number like "72372224 bytes" and get any 1153 meaningful information out of it. It would be more useful to see something 1154 like "69.02 MB". That's what this function does. Any time you want to display 1155 a byte value, i.e.:: 1156 1157 print "Size: %s bytes" % bytes 1158 1159 Call this function instead:: 1160 1161 print "Size: %s" % displayBytes(bytes) 1162 1163 What comes out will be sensibly formatted. The indicated number of digits 1164 will be listed after the decimal point, rounded based on whatever rules are 1165 used by Python's standard C{%f} string format specifier. (Values less than 1 1166 kB will be listed in bytes and will not have a decimal point, since the 1167 concept of a fractional byte is nonsensical.) 1168 1169 @param bytes: Byte quantity. 1170 @type bytes: Integer number of bytes. 1171 1172 @param digits: Number of digits to display after the decimal point. 1173 @type digits: Integer value, typically 2-5. 1174 1175 @return: String, formatted for sensible display. 1176 """ 1177 if bytes is None: 1178 raise ValueError("Cannot display byte value of None.") 1179 bytes = float(bytes) 1180 if math.fabs(bytes) < BYTES_PER_KBYTE: 1181 fmt = "%.0f bytes" 1182 value = bytes 1183 elif math.fabs(bytes) < BYTES_PER_MBYTE: 1184 fmt = "%." + "%d" % digits + "f kB" 1185 value = bytes / BYTES_PER_KBYTE 1186 elif math.fabs(bytes) < BYTES_PER_GBYTE: 1187 fmt = "%." + "%d" % digits + "f MB" 1188 value = bytes / BYTES_PER_MBYTE 1189 else: 1190 fmt = "%." + "%d" % digits + "f GB" 1191 value = bytes / BYTES_PER_GBYTE 1192 return fmt % value 1193
1194 1195 ################################## 1196 # getFunctionReference() function 1197 ################################## 1198 1199 -def getFunctionReference(module, function):
1200 """ 1201 Gets a reference to a named function. 1202 1203 This does some hokey-pokey to get back a reference to a dynamically named 1204 function. For instance, say you wanted to get a reference to the 1205 C{os.path.isdir} function. You could use:: 1206 1207 myfunc = getFunctionReference("os.path", "isdir") 1208 1209 Although we won't bomb out directly, behavior is pretty much undefined if 1210 you pass in C{None} or C{""} for either C{module} or C{function}. 1211 1212 The only validation we enforce is that whatever we get back must be 1213 callable. 1214 1215 I derived this code based on the internals of the Python unittest 1216 implementation. I don't claim to completely understand how it works. 1217 1218 @param module: Name of module associated with function. 1219 @type module: Something like "os.path" or "CedarBackup3.util" 1220 1221 @param function: Name of function 1222 @type function: Something like "isdir" or "getUidGid" 1223 1224 @return: Reference to function associated with name. 1225 1226 @raise ImportError: If the function cannot be found. 1227 @raise ValueError: If the resulting reference is not callable. 1228 1229 @copyright: Some of this code, prior to customization, was originally part 1230 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python 1231 Software Foundation; All Rights Reserved. 1232 """ 1233 parts = [] 1234 if module is not None and module != "": 1235 parts = module.split(".") 1236 if function is not None and function != "": 1237 parts.append(function) 1238 copy = parts[:] 1239 while copy: 1240 try: 1241 module = __import__(".".join(copy)) 1242 break 1243 except ImportError: 1244 del copy[-1] 1245 if not copy: raise 1246 parts = parts[1:] 1247 obj = module 1248 for part in parts: 1249 obj = getattr(obj, part) 1250 if not isinstance(obj, collections.Callable): 1251 raise ValueError("Reference to %s.%s is not callable." % (module, function)) 1252 return obj
1253
1254 1255 ####################### 1256 # getUidGid() function 1257 ####################### 1258 1259 -def getUidGid(user, group):
1260 """ 1261 Get the uid/gid associated with a user/group pair 1262 1263 This is a no-op if user/group functionality is not available on the platform. 1264 1265 @param user: User name 1266 @type user: User name as a string 1267 1268 @param group: Group name 1269 @type group: Group name as a string 1270 1271 @return: Tuple C{(uid, gid)} matching passed-in user and group. 1272 @raise ValueError: If the ownership user/group values are invalid 1273 """ 1274 if _UID_GID_AVAILABLE: 1275 try: 1276 uid = pwd.getpwnam(user)[2] 1277 gid = grp.getgrnam(group)[2] 1278 return (uid, gid) 1279 except Exception as e: 1280 logger.debug("Error looking up uid and gid for [%s:%s]: %s", user, group, e) 1281 raise ValueError("Unable to lookup up uid and gid for passed in user/group.") 1282 else: 1283 return (0, 0)
1284
1285 1286 ############################# 1287 # changeOwnership() function 1288 ############################# 1289 1290 -def changeOwnership(path, user, group):
1291 """ 1292 Changes ownership of path to match the user and group. 1293 1294 This is a no-op if user/group functionality is not available on the 1295 platform, or if the either passed-in user or group is C{None}. Further, we 1296 won't even try to do it unless running as root, since it's unlikely to work. 1297 1298 @param path: Path whose ownership to change. 1299 @param user: User which owns file. 1300 @param group: Group which owns file. 1301 """ 1302 if _UID_GID_AVAILABLE: 1303 if user is None or group is None: 1304 logger.debug("User or group is None, so not attempting to change owner on [%s].", path) 1305 elif not isRunningAsRoot(): 1306 logger.debug("Not root, so not attempting to change owner on [%s].", path) 1307 else: 1308 try: 1309 (uid, gid) = getUidGid(user, group) 1310 os.chown(path, uid, gid) 1311 except Exception as e: 1312 logger.error("Error changing ownership of [%s]: %s", path, e)
1313
1314 1315 ############################# 1316 # isRunningAsRoot() function 1317 ############################# 1318 1319 -def isRunningAsRoot():
1320 """ 1321 Indicates whether the program is running as the root user. 1322 """ 1323 return os.getuid() == 0
1324
1325 1326 ############################## 1327 # splitCommandLine() function 1328 ############################## 1329 1330 -def splitCommandLine(commandLine):
1331 """ 1332 Splits a command line string into a list of arguments. 1333 1334 Unfortunately, there is no "standard" way to parse a command line string, 1335 and it's actually not an easy problem to solve portably (essentially, we 1336 have to emulate the shell argument-processing logic). This code only 1337 respects double quotes (C{"}) for grouping arguments, not single quotes 1338 (C{'}). Make sure you take this into account when building your command 1339 line. 1340 1341 Incidentally, I found this particular parsing method while digging around in 1342 Google Groups, and I tweaked it for my own use. 1343 1344 @param commandLine: Command line string 1345 @type commandLine: String, i.e. "cback3 --verbose stage store" 1346 1347 @return: List of arguments, suitable for passing to C{popen2}. 1348 1349 @raise ValueError: If the command line is None. 1350 """ 1351 if commandLine is None: 1352 raise ValueError("Cannot split command line of None.") 1353 fields = re.findall('[^ "]+|"[^"]+"', commandLine) 1354 fields = [field.replace('"', '') for field in fields] 1355 return fields
1356
1357 1358 ############################ 1359 # resolveCommand() function 1360 ############################ 1361 1362 -def resolveCommand(command):
1363 """ 1364 Resolves the real path to a command through the path resolver mechanism. 1365 1366 Both extensions and standard Cedar Backup functionality need a way to 1367 resolve the "real" location of various executables. Normally, they assume 1368 that these executables are on the system path, but some callers need to 1369 specify an alternate location. 1370 1371 Ideally, we want to handle this configuration in a central location. The 1372 Cedar Backup path resolver mechanism (a singleton called 1373 L{PathResolverSingleton}) provides the central location to store the 1374 mappings. This function wraps access to the singleton, and is what all 1375 functions (extensions or standard functionality) should call if they need to 1376 find a command. 1377 1378 The passed-in command must actually be a list, in the standard form used by 1379 all existing Cedar Backup code (something like C{["svnlook", ]}). The 1380 lookup will actually be done on the first element in the list, and the 1381 returned command will always be in list form as well. 1382 1383 If the passed-in command can't be resolved or no mapping exists, then the 1384 command itself will be returned unchanged. This way, we neatly fall back on 1385 default behavior if we have no sensible alternative. 1386 1387 @param command: Command to resolve. 1388 @type command: List form of command, i.e. C{["svnlook", ]}. 1389 1390 @return: Path to command or just command itself if no mapping exists. 1391 """ 1392 singleton = PathResolverSingleton.getInstance() 1393 name = command[0] 1394 result = command[:] 1395 result[0] = singleton.lookup(name, name) 1396 return result
1397
1398 1399 ############################ 1400 # executeCommand() function 1401 ############################ 1402 1403 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1404 """ 1405 Executes a shell command, hopefully in a safe way. 1406 1407 This function exists to replace direct calls to C{os.popen} in the Cedar 1408 Backup code. It's not safe to call a function such as C{os.popen()} with 1409 untrusted arguments, since that can cause problems if the string contains 1410 non-safe variables or other constructs (imagine that the argument is 1411 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/; 1412 echo"} in the current environment). 1413 1414 Instead, it's safer to pass a list of arguments in the style supported bt 1415 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe} 1416 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}. 1417 1418 Under the normal case, this function will return a tuple of C{(status, 1419 None)} where the status is the wait-encoded return status of the call per 1420 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as 1421 C{True}, the function will return a tuple of C{(status, output)} where 1422 C{output} is a list of strings, one entry per line in the output from the 1423 command. Output is always logged to the C{outputLogger.info()} target, 1424 regardless of whether it's returned. 1425 1426 By default, C{stdout} and C{stderr} will be intermingled in the output. 1427 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be 1428 included in the output. 1429 1430 The C{doNotLog} parameter exists so that callers can force the function to 1431 not log command output to the debug log. Normally, you would want to log. 1432 However, if you're using this function to write huge output files (i.e. 1433 database backups written to C{stdout}) then you might want to avoid putting 1434 all that information into the debug log. 1435 1436 The C{outputFile} parameter exists to make it easier for a caller to push 1437 output into a file, i.e. as a substitute for redirection to a file. If this 1438 value is passed in, each time a line of output is generated, it will be 1439 written to the file using C{outputFile.write()}. At the end, the file 1440 descriptor will be flushed using C{outputFile.flush()}. The caller 1441 maintains responsibility for closing the file object appropriately. 1442 1443 @note: I know that it's a bit confusing that the command and the arguments 1444 are both lists. I could have just required the caller to pass in one big 1445 list. However, I think it makes some sense to keep the command (the 1446 constant part of what we're executing, i.e. C{"scp -B"}) separate from its 1447 arguments, even if they both end up looking kind of similar. 1448 1449 @note: You cannot redirect output via shell constructs (i.e. C{>file}, 1450 C{2>/dev/null}, etc.) using this function. The redirection string would be 1451 passed to the command just like any other argument. However, you can 1452 implement the equivalent to redirection using C{ignoreStderr} and 1453 C{outputFile}, as discussed above. 1454 1455 @note: The operating system environment is partially sanitized before 1456 the command is invoked. See L{sanitizeEnvironment} for details. 1457 1458 @param command: Shell command to execute 1459 @type command: List of individual arguments that make up the command 1460 1461 @param args: List of arguments to the command 1462 @type args: List of additional arguments to the command 1463 1464 @param returnOutput: Indicates whether to return the output of the command 1465 @type returnOutput: Boolean C{True} or C{False} 1466 1467 @param ignoreStderr: Whether stderr should be discarded 1468 @type ignoreStderr: Boolean True or False 1469 1470 @param doNotLog: Indicates that output should not be logged. 1471 @type doNotLog: Boolean C{True} or C{False} 1472 1473 @param outputFile: File object that all output should be written to. 1474 @type outputFile: File object as returned from C{open()} or C{file()}, configured for binary write 1475 1476 @return: Tuple of C{(result, output)} as described above. 1477 """ 1478 logger.debug("Executing command %s with args %s.", command, args) 1479 outputLogger.info("Executing command %s with args %s.", command, args) 1480 if doNotLog: 1481 logger.debug("Note: output will not be logged, per the doNotLog flag.") 1482 outputLogger.info("Note: output will not be logged, per the doNotLog flag.") 1483 output = [] 1484 fields = command[:] # make sure to copy it so we don't destroy it 1485 fields.extend(args) 1486 try: 1487 sanitizeEnvironment() # make sure we have a consistent environment 1488 try: 1489 pipe = Pipe(fields, ignoreStderr=ignoreStderr) 1490 except OSError: 1491 # On some platforms (i.e. Cygwin) this intermittently fails the first time we do it. 1492 # So, we attempt it a second time and if that works, we just go on as usual. 1493 # The problem appears to be that we sometimes get a bad stderr file descriptor. 1494 pipe = Pipe(fields, ignoreStderr=ignoreStderr) 1495 while True: 1496 line = pipe.stdout.readline() 1497 if not line: break 1498 if returnOutput: output.append(line.decode("utf-8")) 1499 if outputFile is not None: outputFile.write(line) 1500 if not doNotLog: outputLogger.info(line.decode("utf-8")[:-1]) # this way the log will (hopefully) get updated in realtime 1501 if outputFile is not None: 1502 try: # note, not every file-like object can be flushed 1503 outputFile.flush() 1504 except: pass 1505 if returnOutput: 1506 return (pipe.wait(), output) 1507 else: 1508 return (pipe.wait(), None) 1509 except OSError as e: 1510 try: 1511 if returnOutput: 1512 if output != []: 1513 return (pipe.wait(), output) 1514 else: 1515 return (pipe.wait(), [ e, ]) 1516 else: 1517 return (pipe.wait(), None) 1518 except UnboundLocalError: # pipe not set 1519 if returnOutput: 1520 return (256, []) 1521 else: 1522 return (256, None)
1523
1524 1525 ############################## 1526 # calculateFileAge() function 1527 ############################## 1528 1529 -def calculateFileAge(path):
1530 """ 1531 Calculates the age (in days) of a file. 1532 1533 The "age" of a file is the amount of time since the file was last used, per 1534 the most recent of the file's C{st_atime} and C{st_mtime} values. 1535 1536 Technically, we only intend this function to work with files, but it will 1537 probably work with anything on the filesystem. 1538 1539 @param path: Path to a file on disk. 1540 1541 @return: Age of the file in days (possibly fractional). 1542 @raise OSError: If the file doesn't exist. 1543 """ 1544 currentTime = int(time.time()) 1545 fileStats = os.stat(path) 1546 lastUse = max(fileStats.st_atime, fileStats.st_mtime) # "most recent" is "largest" 1547 ageInSeconds = currentTime - lastUse 1548 ageInDays = ageInSeconds / SECONDS_PER_DAY 1549 return ageInDays
1550
1551 1552 ################### 1553 # mount() function 1554 ################### 1555 1556 -def mount(devicePath, mountPoint, fsType):
1557 """ 1558 Mounts the indicated device at the indicated mount point. 1559 1560 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount 1561 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any 1562 filesystem type that is supported by C{mount} on your platform. If the type 1563 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may 1564 not work on all systems. 1565 1566 @note: This only works on platforms that have a concept of "mounting" a 1567 filesystem through a command-line C{"mount"} command, like UNIXes. It 1568 won't work on Windows. 1569 1570 @param devicePath: Path of device to be mounted. 1571 @param mountPoint: Path that device should be mounted at. 1572 @param fsType: Type of the filesystem assumed to be available via the device. 1573 1574 @raise IOError: If the device cannot be mounted. 1575 """ 1576 if fsType is None: 1577 args = [ devicePath, mountPoint ] 1578 else: 1579 args = [ "-t", fsType, devicePath, mountPoint ] 1580 command = resolveCommand(MOUNT_COMMAND) 1581 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0] 1582 if result != 0: 1583 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1584
1585 1586 ##################### 1587 # unmount() function 1588 ##################### 1589 1590 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1591 """ 1592 Unmounts whatever device is mounted at the indicated mount point. 1593 1594 Sometimes, it might not be possible to unmount the mount point immediately, 1595 if there are still files open there. Use the C{attempts} and C{waitSeconds} 1596 arguments to indicate how many unmount attempts to make and how many seconds 1597 to wait between attempts. If you pass in zero attempts, no attempts will be 1598 made (duh). 1599 1600 If the indicated mount point is not really a mount point per 1601 C{os.path.ismount()}, then it will be ignored. This seems to be a safer 1602 check then looking through C{/etc/mtab}, since C{ismount()} is already in 1603 the Python standard library and is documented as working on all POSIX 1604 systems. 1605 1606 If C{removeAfter} is C{True}, then the mount point will be removed using 1607 C{os.rmdir()} after the unmount action succeeds. If for some reason the 1608 mount point is not a directory, then it will not be removed. 1609 1610 @note: This only works on platforms that have a concept of "mounting" a 1611 filesystem through a command-line C{"mount"} command, like UNIXes. It 1612 won't work on Windows. 1613 1614 @param mountPoint: Mount point to be unmounted. 1615 @param removeAfter: Remove the mount point after unmounting it. 1616 @param attempts: Number of times to attempt the unmount. 1617 @param waitSeconds: Number of seconds to wait between repeated attempts. 1618 1619 @raise IOError: If the mount point is still mounted after attempts are exhausted. 1620 """ 1621 if os.path.ismount(mountPoint): 1622 for attempt in range(0, attempts): 1623 logger.debug("Making attempt %d to unmount [%s].", attempt, mountPoint) 1624 command = resolveCommand(UMOUNT_COMMAND) 1625 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0] 1626 if result != 0: 1627 logger.error("Error [%d] unmounting [%s] on attempt %d.", result, mountPoint, attempt) 1628 elif os.path.ismount(mountPoint): 1629 logger.error("After attempt %d, [%s] is still mounted.", attempt, mountPoint) 1630 else: 1631 logger.debug("Successfully unmounted [%s] on attempt %d.", mountPoint, attempt) 1632 break # this will cause us to skip the loop else: clause 1633 if attempt+1 < attempts: # i.e. this isn't the last attempt 1634 if waitSeconds > 0: 1635 logger.info("Sleeping %d second(s) before next unmount attempt.", waitSeconds) 1636 time.sleep(waitSeconds) 1637 else: 1638 if os.path.ismount(mountPoint): 1639 raise IOError("Unable to unmount [%s] after %d attempts.", mountPoint, attempts) 1640 logger.info("Mount point [%s] seems to have finally gone away.", mountPoint) 1641 if os.path.isdir(mountPoint) and removeAfter: 1642 logger.debug("Removing mount point [%s].", mountPoint) 1643 os.rmdir(mountPoint)
1644
1645 1646 ########################### 1647 # deviceMounted() function 1648 ########################### 1649 1650 -def deviceMounted(devicePath):
1651 """ 1652 Indicates whether a specific filesystem device is currently mounted. 1653 1654 We determine whether the device is mounted by looking through the system's 1655 C{mtab} file. This file shows every currently-mounted filesystem, ordered 1656 by device. We only do the check if the C{mtab} file exists and is readable. 1657 Otherwise, we assume that the device is not mounted. 1658 1659 @note: This only works on platforms that have a concept of an mtab file 1660 to show mounted volumes, like UNIXes. It won't work on Windows. 1661 1662 @param devicePath: Path of device to be checked 1663 1664 @return: True if device is mounted, false otherwise. 1665 """ 1666 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK): 1667 realPath = os.path.realpath(devicePath) 1668 with open(MTAB_FILE) as f: 1669 lines = f.readlines() 1670 for line in lines: 1671 (mountDevice, mountPoint, remainder) = line.split(None, 2) 1672 if mountDevice in [ devicePath, realPath, ]: 1673 logger.debug("Device [%s] is mounted at [%s].", devicePath, mountPoint) 1674 return True 1675 return False
1676
1677 1678 ######################## 1679 # encodePath() function 1680 ######################## 1681 1682 -def encodePath(path):
1683 """ 1684 Safely encodes a filesystem path as a Unicode string, converting bytes to fileystem encoding if necessary. 1685 @param path: Path to encode 1686 @return: Path, as a string, encoded appropriately 1687 @raise ValueError: If the path cannot be encoded properly. 1688 @see: http://lucumr.pocoo.org/2013/7/2/the-updated-guide-to-unicode/ 1689 """ 1690 if path is None: 1691 return path 1692 try: 1693 if isinstance(path, bytes): 1694 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() 1695 path = path.decode(encoding, "surrogateescape") # to match what os.listdir() does 1696 return path 1697 except UnicodeError as e: 1698 raise ValueError("Path could not be safely encoded as %s: %s" % (encoding, str(e)))
1699
1700 1701 ######################## 1702 # nullDevice() function 1703 ######################## 1704 1705 -def nullDevice():
1706 """ 1707 Attempts to portably return the null device on this system. 1708 1709 The null device is something like C{/dev/null} on a UNIX system. The name 1710 varies on other platforms. 1711 """ 1712 return os.devnull
1713
1714 1715 ############################## 1716 # deriveDayOfWeek() function 1717 ############################## 1718 1719 -def deriveDayOfWeek(dayName):
1720 """ 1721 Converts English day name to numeric day of week as from C{time.localtime}. 1722 1723 For instance, the day C{monday} would be converted to the number C{0}. 1724 1725 @param dayName: Day of week to convert 1726 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc. 1727 1728 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible. 1729 """ 1730 if dayName.lower() == "monday": 1731 return 0 1732 elif dayName.lower() == "tuesday": 1733 return 1 1734 elif dayName.lower() == "wednesday": 1735 return 2 1736 elif dayName.lower() == "thursday": 1737 return 3 1738 elif dayName.lower() == "friday": 1739 return 4 1740 elif dayName.lower() == "saturday": 1741 return 5 1742 elif dayName.lower() == "sunday": 1743 return 6 1744 else: 1745 return -1 # What else can we do?? Thrown an exception, I guess.
1746
1747 1748 ########################### 1749 # isStartOfWeek() function 1750 ########################### 1751 1752 -def isStartOfWeek(startingDay):
1753 """ 1754 Indicates whether "today" is the backup starting day per configuration. 1755 1756 If the current day's English name matches the indicated starting day, then 1757 today is a starting day. 1758 1759 @param startingDay: Configured starting day. 1760 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc. 1761 1762 @return: Boolean indicating whether today is the starting day. 1763 """ 1764 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay) 1765 if value: 1766 logger.debug("Today is the start of the week.") 1767 else: 1768 logger.debug("Today is NOT the start of the week.") 1769 return value
1770
1771 1772 ################################# 1773 # buildNormalizedPath() function 1774 ################################# 1775 1776 -def buildNormalizedPath(path):
1777 """ 1778 Returns a "normalized" path based on a path name. 1779 1780 A normalized path is a representation of a path that is also a valid file 1781 name. To make a valid file name out of a complete path, we have to convert 1782 or remove some characters that are significant to the filesystem -- in 1783 particular, the path separator and any leading C{'.'} character (which would 1784 cause the file to be hidden in a file listing). 1785 1786 Note that this is a one-way transformation -- you can't safely derive the 1787 original path from the normalized path. 1788 1789 To normalize a path, we begin by looking at the first character. If the 1790 first character is C{'/'} or C{'\\'}, it gets removed. If the first 1791 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the 1792 rest of the path and convert all remaining C{'/'} or C{'\\'} characters 1793 C{'-'}, and all remaining whitespace characters to C{'_'}. 1794 1795 As a special case, a path consisting only of a single C{'/'} or C{'\\'} 1796 character will be converted to C{'-'}. 1797 1798 @param path: Path to normalize 1799 1800 @return: Normalized path as described above. 1801 1802 @raise ValueError: If the path is None 1803 """ 1804 if path is None: 1805 raise ValueError("Cannot normalize path None.") 1806 elif len(path) == 0: 1807 return path 1808 elif path == "/" or path == "\\": 1809 return "-" 1810 else: 1811 normalized = path 1812 normalized = re.sub(r"^\/", "", normalized) # remove leading '/' 1813 normalized = re.sub(r"^\\", "", normalized) # remove leading '\' 1814 normalized = re.sub(r"^\.", "_", normalized) # convert leading '.' to '_' so file won't be hidden 1815 normalized = re.sub(r"\/", "-", normalized) # convert all '/' characters to '-' 1816 normalized = re.sub(r"\\", "-", normalized) # convert all '\' characters to '-' 1817 normalized = re.sub(r"\s", "_", normalized) # convert all whitespace to '_' 1818 return normalized
1819
1820 1821 ################################# 1822 # sanitizeEnvironment() function 1823 ################################# 1824 1825 -def sanitizeEnvironment():
1826 """ 1827 Sanitizes the operating system environment. 1828 1829 The operating system environment is contained in C{os.environ}. This method 1830 sanitizes the contents of that dictionary. 1831 1832 Currently, all it does is reset the locale (removing C{$LC_*}) and set the 1833 default language (C{$LANG}) to L{DEFAULT_LANGUAGE}. This way, we can count 1834 on consistent localization regardless of what the end-user has configured. 1835 This is important for code that needs to parse program output. 1836 1837 The C{os.environ} dictionary is modifed in-place. If C{$LANG} is already 1838 set to the proper value, it is not re-set, so we can avoid the memory leaks 1839 that are documented to occur on BSD-based systems. 1840 1841 @return: Copy of the sanitized environment. 1842 """ 1843 for var in LOCALE_VARS: 1844 if var in os.environ: 1845 del os.environ[var] 1846 if LANG_VAR in os.environ: 1847 if os.environ[LANG_VAR] != DEFAULT_LANGUAGE: # no need to reset if it exists (avoid leaks on BSD systems) 1848 os.environ[LANG_VAR] = DEFAULT_LANGUAGE 1849 return os.environ.copy()
1850 1869
1870 1871 ######################### 1872 # checkUnique() function 1873 ######################### 1874 1875 -def checkUnique(prefix, values):
1876 """ 1877 Checks that all values are unique. 1878 1879 The values list is checked for duplicate values. If there are 1880 duplicates, an exception is thrown. All duplicate values are listed in 1881 the exception. 1882 1883 @param prefix: Prefix to use in the thrown exception 1884 @param values: List of values to check 1885 1886 @raise ValueError: If there are duplicates in the list 1887 """ 1888 values.sort() 1889 duplicates = [] 1890 for i in range(1, len(values)): 1891 if values[i-1] == values[i]: 1892 duplicates.append(values[i]) 1893 if duplicates: 1894 raise ValueError("%s %s" % (prefix, duplicates))
1895
1896 1897 ####################################### 1898 # parseCommaSeparatedString() function 1899 ####################################### 1900 1901 -def parseCommaSeparatedString(commaString):
1902 """ 1903 Parses a list of values out of a comma-separated string. 1904 1905 The items in the list are split by comma, and then have whitespace 1906 stripped. As a special case, if C{commaString} is C{None}, then C{None} 1907 will be returned. 1908 1909 @param commaString: List of values in comma-separated string format. 1910 @return: Values from commaString split into a list, or C{None}. 1911 """ 1912 if commaString is None: 1913 return None 1914 else: 1915 pass1 = commaString.split(",") 1916 pass2 = [] 1917 for item in pass1: 1918 item = item.strip() 1919 if len(item) > 0: 1920 pass2.append(item) 1921 return pass2
1922