1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Provides backup peer-related objects and utility functions.
41
42 @sort: LocalPeer, RemotePeer
43
44 @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file.
45 @var DEF_STAGE_INDICATOR: Name of the default stage indicator file.
46
47 @author: Kenneth J. Pronovici <pronovic@ieee.org>
48 """
49
50
51
52
53
54
55
56 import os
57 import logging
58 import shutil
59
60
61 from CedarBackup2.filesystem import FilesystemList
62 from CedarBackup2.util import resolveCommand, executeCommand, isRunningAsRoot
63 from CedarBackup2.util import splitCommandLine, encodePath
64 from CedarBackup2.config import VALID_FAILURE_MODES
65
66
67
68
69
70
71 logger = logging.getLogger("CedarBackup2.log.peer")
72
73 DEF_RCP_COMMAND = [ "/usr/bin/scp", "-B", "-q", "-C" ]
74 DEF_RSH_COMMAND = [ "/usr/bin/ssh", ]
75 DEF_CBACK_COMMAND = "/usr/bin/cback"
76
77 DEF_COLLECT_INDICATOR = "cback.collect"
78 DEF_STAGE_INDICATOR = "cback.stage"
79
80 SU_COMMAND = [ "su" ]
88
89
90
91
92
93 """
94 Backup peer representing a local peer in a backup pool.
95
96 This is a class representing a local (non-network) peer in a backup pool.
97 Local peers are backed up by simple filesystem copy operations. A local
98 peer has associated with it a name (typically, but not necessarily, a
99 hostname) and a collect directory.
100
101 The public methods other than the constructor are part of a "backup peer"
102 interface shared with the C{RemotePeer} class.
103
104 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
105 _copyLocalDir, _copyLocalFile, name, collectDir
106 """
107
108
109
110
111
112 - def __init__(self, name, collectDir, ignoreFailureMode=None):
113 """
114 Initializes a local backup peer.
115
116 Note that the collect directory must be an absolute path, but does not
117 have to exist when the object is instantiated. We do a lazy validation
118 on this value since we could (potentially) be creating peer objects
119 before an ongoing backup completed.
120
121 @param name: Name of the backup peer
122 @type name: String, typically a hostname
123
124 @param collectDir: Path to the peer's collect directory
125 @type collectDir: String representing an absolute local path on disk
126
127 @param ignoreFailureMode: Ignore failure mode for this peer
128 @type ignoreFailureMode: One of VALID_FAILURE_MODES
129
130 @raise ValueError: If the name is empty.
131 @raise ValueError: If collect directory is not an absolute path.
132 """
133 self._name = None
134 self._collectDir = None
135 self._ignoreFailureMode = None
136 self.name = name
137 self.collectDir = collectDir
138 self.ignoreFailureMode = ignoreFailureMode
139
140
141
142
143
144
146 """
147 Property target used to set the peer name.
148 The value must be a non-empty string and cannot be C{None}.
149 @raise ValueError: If the value is an empty string or C{None}.
150 """
151 if value is None or len(value) < 1:
152 raise ValueError("Peer name must be a non-empty string.")
153 self._name = value
154
156 """
157 Property target used to get the peer name.
158 """
159 return self._name
160
162 """
163 Property target used to set the collect directory.
164 The value must be an absolute path and cannot be C{None}.
165 It does not have to exist on disk at the time of assignment.
166 @raise ValueError: If the value is C{None} or is not an absolute path.
167 @raise ValueError: If a path cannot be encoded properly.
168 """
169 if value is None or not os.path.isabs(value):
170 raise ValueError("Collect directory must be an absolute path.")
171 self._collectDir = encodePath(value)
172
174 """
175 Property target used to get the collect directory.
176 """
177 return self._collectDir
178
180 """
181 Property target used to set the ignoreFailure mode.
182 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
183 @raise ValueError: If the value is not valid.
184 """
185 if value is not None:
186 if value not in VALID_FAILURE_MODES:
187 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
188 self._ignoreFailureMode = value
189
191 """
192 Property target used to get the ignoreFailure mode.
193 """
194 return self._ignoreFailureMode
195
196 name = property(_getName, _setName, None, "Name of the peer.")
197 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
198 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
199
200
201
202
203
204
205 - def stagePeer(self, targetDir, ownership=None, permissions=None):
206 """
207 Stages data from the peer into the indicated local target directory.
208
209 The collect and target directories must both already exist before this
210 method is called. If passed in, ownership and permissions will be
211 applied to the files that are copied.
212
213 @note: The caller is responsible for checking that the indicator exists,
214 if they care. This function only stages the files within the directory.
215
216 @note: If you have user/group as strings, call the L{util.getUidGid} function
217 to get the associated uid/gid as an ownership tuple.
218
219 @param targetDir: Target directory to write data into
220 @type targetDir: String representing a directory on disk
221
222 @param ownership: Owner and group that the staged files should have
223 @type ownership: Tuple of numeric ids C{(uid, gid)}
224
225 @param permissions: Permissions that the staged files should have
226 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
227
228 @return: Number of files copied from the source directory to the target directory.
229
230 @raise ValueError: If collect directory is not a directory or does not exist
231 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
232 @raise ValueError: If a path cannot be encoded properly.
233 @raise IOError: If there were no files to stage (i.e. the directory was empty)
234 @raise IOError: If there is an IO error copying a file.
235 @raise OSError: If there is an OS error copying or changing permissions on a file
236 """
237 targetDir = encodePath(targetDir)
238 if not os.path.isabs(targetDir):
239 logger.debug("Target directory [%s] not an absolute path." % targetDir)
240 raise ValueError("Target directory must be an absolute path.")
241 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
242 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
243 raise ValueError("Collect directory is not a directory or does not exist on disk.")
244 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
245 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
246 raise ValueError("Target directory is not a directory or does not exist on disk.")
247 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions)
248 if count == 0:
249 raise IOError("Did not copy any files from local peer.")
250 return count
251
253 """
254 Checks the collect indicator in the peer's staging directory.
255
256 When a peer has completed collecting its backup files, it will write an
257 empty indicator file into its collect directory. This method checks to
258 see whether that indicator has been written. We're "stupid" here - if
259 the collect directory doesn't exist, you'll naturally get back C{False}.
260
261 If you need to, you can override the name of the collect indicator file
262 by passing in a different name.
263
264 @param collectIndicator: Name of the collect indicator file to check
265 @type collectIndicator: String representing name of a file in the collect directory
266
267 @return: Boolean true/false depending on whether the indicator exists.
268 @raise ValueError: If a path cannot be encoded properly.
269 """
270 collectIndicator = encodePath(collectIndicator)
271 if collectIndicator is None:
272 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR))
273 else:
274 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
275
277 """
278 Writes the stage indicator in the peer's staging directory.
279
280 When the master has completed collecting its backup files, it will write
281 an empty indicator file into the peer's collect directory. The presence
282 of this file implies that the staging process is complete.
283
284 If you need to, you can override the name of the stage indicator file by
285 passing in a different name.
286
287 @note: If you have user/group as strings, call the L{util.getUidGid}
288 function to get the associated uid/gid as an ownership tuple.
289
290 @param stageIndicator: Name of the indicator file to write
291 @type stageIndicator: String representing name of a file in the collect directory
292
293 @param ownership: Owner and group that the indicator file should have
294 @type ownership: Tuple of numeric ids C{(uid, gid)}
295
296 @param permissions: Permissions that the indicator file should have
297 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
298
299 @raise ValueError: If collect directory is not a directory or does not exist
300 @raise ValueError: If a path cannot be encoded properly.
301 @raise IOError: If there is an IO error creating the file.
302 @raise OSError: If there is an OS error creating or changing permissions on the file
303 """
304 stageIndicator = encodePath(stageIndicator)
305 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
306 logger.debug("Collect directory [%s] is not a directory or does not exist on disk." % self.collectDir)
307 raise ValueError("Collect directory is not a directory or does not exist on disk.")
308 if stageIndicator is None:
309 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
310 else:
311 fileName = os.path.join(self.collectDir, stageIndicator)
312 LocalPeer._copyLocalFile(None, fileName, ownership, permissions)
313
314
315
316
317
318
319 @staticmethod
320 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
321 """
322 Copies files from the source directory to the target directory.
323
324 This function is not recursive. Only the files in the directory will be
325 copied. Ownership and permissions will be left at their default values
326 if new values are not specified. The source and target directories are
327 allowed to be soft links to a directory, but besides that soft links are
328 ignored.
329
330 @note: If you have user/group as strings, call the L{util.getUidGid}
331 function to get the associated uid/gid as an ownership tuple.
332
333 @param sourceDir: Source directory
334 @type sourceDir: String representing a directory on disk
335
336 @param targetDir: Target directory
337 @type targetDir: String representing a directory on disk
338
339 @param ownership: Owner and group that the copied files should have
340 @type ownership: Tuple of numeric ids C{(uid, gid)}
341
342 @param permissions: Permissions that the staged files should have
343 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
344
345 @return: Number of files copied from the source directory to the target directory.
346
347 @raise ValueError: If source or target is not a directory or does not exist.
348 @raise ValueError: If a path cannot be encoded properly.
349 @raise IOError: If there is an IO error copying the files.
350 @raise OSError: If there is an OS error copying or changing permissions on a files
351 """
352 filesCopied = 0
353 sourceDir = encodePath(sourceDir)
354 targetDir = encodePath(targetDir)
355 for fileName in os.listdir(sourceDir):
356 sourceFile = os.path.join(sourceDir, fileName)
357 targetFile = os.path.join(targetDir, fileName)
358 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions)
359 filesCopied += 1
360 return filesCopied
361
362 @staticmethod
363 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
364 """
365 Copies a source file to a target file.
366
367 If the source file is C{None} then the target file will be created or
368 overwritten as an empty file. If the target file is C{None}, this method
369 is a no-op. Attempting to copy a soft link or a directory will result in
370 an exception.
371
372 @note: If you have user/group as strings, call the L{util.getUidGid}
373 function to get the associated uid/gid as an ownership tuple.
374
375 @note: We will not overwrite a target file that exists when this method
376 is invoked. If the target already exists, we'll raise an exception.
377
378 @param sourceFile: Source file to copy
379 @type sourceFile: String representing a file on disk, as an absolute path
380
381 @param targetFile: Target file to create
382 @type targetFile: String representing a file on disk, as an absolute path
383
384 @param ownership: Owner and group that the copied should have
385 @type ownership: Tuple of numeric ids C{(uid, gid)}
386
387 @param permissions: Permissions that the staged files should have
388 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
389
390 @param overwrite: Indicates whether it's OK to overwrite the target file.
391 @type overwrite: Boolean true/false.
392
393 @raise ValueError: If the passed-in source file is not a regular file.
394 @raise ValueError: If a path cannot be encoded properly.
395 @raise IOError: If the target file already exists.
396 @raise IOError: If there is an IO error copying the file
397 @raise OSError: If there is an OS error copying or changing permissions on a file
398 """
399 targetFile = encodePath(targetFile)
400 sourceFile = encodePath(sourceFile)
401 if targetFile is None:
402 return
403 if not overwrite:
404 if os.path.exists(targetFile):
405 raise IOError("Target file [%s] already exists." % targetFile)
406 if sourceFile is None:
407 open(targetFile, "w").write("")
408 else:
409 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile):
410 shutil.copy(sourceFile, targetFile)
411 else:
412 logger.debug("Source [%s] is not a regular file." % sourceFile)
413 raise ValueError("Source is not a regular file.")
414 if ownership is not None:
415 os.chown(targetFile, ownership[0], ownership[1])
416 if permissions is not None:
417 os.chmod(targetFile, permissions)
418
425
426
427
428
429
430 """
431 Backup peer representing a remote peer in a backup pool.
432
433 This is a class representing a remote (networked) peer in a backup pool.
434 Remote peers are backed up using an rcp-compatible copy command. A remote
435 peer has associated with it a name (which must be a valid hostname), a
436 collect directory, a working directory and a copy method (an rcp-compatible
437 command).
438
439 You can also set an optional local user value. This username will be used
440 as the local user for any remote copies that are required. It can only be
441 used if the root user is executing the backup. The root user will C{su} to
442 the local user and execute the remote copies as that user.
443
444 The copy method is associated with the peer and not with the actual request
445 to copy, because we can envision that each remote host might have a
446 different connect method.
447
448 The public methods other than the constructor are part of a "backup peer"
449 interface shared with the C{LocalPeer} class.
450
451 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
452 executeRemoteCommand, executeManagedAction, _getDirContents,
453 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir,
454 remoteUser, rcpCommand, rshCommand, cbackCommand
455 """
456
457
458
459
460
461 - def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None,
462 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None,
463 ignoreFailureMode=None):
464 """
465 Initializes a remote backup peer.
466
467 @note: If provided, each command will eventually be parsed into a list of
468 strings suitable for passing to C{util.executeCommand} in order to avoid
469 security holes related to shell interpolation. This parsing will be
470 done by the L{util.splitCommandLine} function. See the documentation for
471 that function for some important notes about its limitations.
472
473 @param name: Name of the backup peer
474 @type name: String, must be a valid DNS hostname
475
476 @param collectDir: Path to the peer's collect directory
477 @type collectDir: String representing an absolute path on the remote peer
478
479 @param workingDir: Working directory that can be used to create temporary files, etc.
480 @type workingDir: String representing an absolute path on the current host.
481
482 @param remoteUser: Name of the Cedar Backup user on the remote peer
483 @type remoteUser: String representing a username, valid via remote shell to the peer
484
485 @param localUser: Name of the Cedar Backup user on the current host
486 @type localUser: String representing a username, valid on the current host
487
488 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
489 @type rcpCommand: String representing a system command including required arguments
490
491 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
492 @type rshCommand: String representing a system command including required arguments
493
494 @param cbackCommand: A chack-compatible command to use for executing managed actions
495 @type cbackCommand: String representing a system command including required arguments
496
497 @param ignoreFailureMode: Ignore failure mode for this peer
498 @type ignoreFailureMode: One of VALID_FAILURE_MODES
499
500 @raise ValueError: If collect directory is not an absolute path
501 """
502 self._name = None
503 self._collectDir = None
504 self._workingDir = None
505 self._remoteUser = None
506 self._localUser = None
507 self._rcpCommand = None
508 self._rcpCommandList = None
509 self._rshCommand = None
510 self._rshCommandList = None
511 self._cbackCommand = None
512 self._ignoreFailureMode = None
513 self.name = name
514 self.collectDir = collectDir
515 self.workingDir = workingDir
516 self.remoteUser = remoteUser
517 self.localUser = localUser
518 self.rcpCommand = rcpCommand
519 self.rshCommand = rshCommand
520 self.cbackCommand = cbackCommand
521 self.ignoreFailureMode = ignoreFailureMode
522
523
524
525
526
527
529 """
530 Property target used to set the peer name.
531 The value must be a non-empty string and cannot be C{None}.
532 @raise ValueError: If the value is an empty string or C{None}.
533 """
534 if value is None or len(value) < 1:
535 raise ValueError("Peer name must be a non-empty string.")
536 self._name = value
537
539 """
540 Property target used to get the peer name.
541 """
542 return self._name
543
545 """
546 Property target used to set the collect directory.
547 The value must be an absolute path and cannot be C{None}.
548 It does not have to exist on disk at the time of assignment.
549 @raise ValueError: If the value is C{None} or is not an absolute path.
550 @raise ValueError: If the value cannot be encoded properly.
551 """
552 if value is not None:
553 if not os.path.isabs(value):
554 raise ValueError("Collect directory must be an absolute path.")
555 self._collectDir = encodePath(value)
556
558 """
559 Property target used to get the collect directory.
560 """
561 return self._collectDir
562
564 """
565 Property target used to set the working directory.
566 The value must be an absolute path and cannot be C{None}.
567 @raise ValueError: If the value is C{None} or is not an absolute path.
568 @raise ValueError: If the value cannot be encoded properly.
569 """
570 if value is not None:
571 if not os.path.isabs(value):
572 raise ValueError("Working directory must be an absolute path.")
573 self._workingDir = encodePath(value)
574
576 """
577 Property target used to get the working directory.
578 """
579 return self._workingDir
580
582 """
583 Property target used to set the remote user.
584 The value must be a non-empty string and cannot be C{None}.
585 @raise ValueError: If the value is an empty string or C{None}.
586 """
587 if value is None or len(value) < 1:
588 raise ValueError("Peer remote user must be a non-empty string.")
589 self._remoteUser = value
590
592 """
593 Property target used to get the remote user.
594 """
595 return self._remoteUser
596
598 """
599 Property target used to set the local user.
600 The value must be a non-empty string if it is not C{None}.
601 @raise ValueError: If the value is an empty string.
602 """
603 if value is not None:
604 if len(value) < 1:
605 raise ValueError("Peer local user must be a non-empty string.")
606 self._localUser = value
607
609 """
610 Property target used to get the local user.
611 """
612 return self._localUser
613
615 """
616 Property target to set the rcp command.
617
618 The value must be a non-empty string or C{None}. Its value is stored in
619 the two forms: "raw" as provided by the client, and "parsed" into a list
620 suitable for being passed to L{util.executeCommand} via
621 L{util.splitCommandLine}.
622
623 However, all the caller will ever see via the property is the actual
624 value they set (which includes seeing C{None}, even if we translate that
625 internally to C{DEF_RCP_COMMAND}). Internally, we should always use
626 C{self._rcpCommandList} if we want the actual command list.
627
628 @raise ValueError: If the value is an empty string.
629 """
630 if value is None:
631 self._rcpCommand = None
632 self._rcpCommandList = DEF_RCP_COMMAND
633 else:
634 if len(value) >= 1:
635 self._rcpCommand = value
636 self._rcpCommandList = splitCommandLine(self._rcpCommand)
637 else:
638 raise ValueError("The rcp command must be a non-empty string.")
639
641 """
642 Property target used to get the rcp command.
643 """
644 return self._rcpCommand
645
647 """
648 Property target to set the rsh command.
649
650 The value must be a non-empty string or C{None}. Its value is stored in
651 the two forms: "raw" as provided by the client, and "parsed" into a list
652 suitable for being passed to L{util.executeCommand} via
653 L{util.splitCommandLine}.
654
655 However, all the caller will ever see via the property is the actual
656 value they set (which includes seeing C{None}, even if we translate that
657 internally to C{DEF_RSH_COMMAND}). Internally, we should always use
658 C{self._rshCommandList} if we want the actual command list.
659
660 @raise ValueError: If the value is an empty string.
661 """
662 if value is None:
663 self._rshCommand = None
664 self._rshCommandList = DEF_RSH_COMMAND
665 else:
666 if len(value) >= 1:
667 self._rshCommand = value
668 self._rshCommandList = splitCommandLine(self._rshCommand)
669 else:
670 raise ValueError("The rsh command must be a non-empty string.")
671
673 """
674 Property target used to get the rsh command.
675 """
676 return self._rshCommand
677
679 """
680 Property target to set the cback command.
681
682 The value must be a non-empty string or C{None}. Unlike the other
683 command, this value is only stored in the "raw" form provided by the
684 client.
685
686 @raise ValueError: If the value is an empty string.
687 """
688 if value is None:
689 self._cbackCommand = None
690 else:
691 if len(value) >= 1:
692 self._cbackCommand = value
693 else:
694 raise ValueError("The cback command must be a non-empty string.")
695
697 """
698 Property target used to get the cback command.
699 """
700 return self._cbackCommand
701
703 """
704 Property target used to set the ignoreFailure mode.
705 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
706 @raise ValueError: If the value is not valid.
707 """
708 if value is not None:
709 if value not in VALID_FAILURE_MODES:
710 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
711 self._ignoreFailureMode = value
712
714 """
715 Property target used to get the ignoreFailure mode.
716 """
717 return self._ignoreFailureMode
718
719 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).")
720 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
721 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).")
722 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.")
723 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.")
724 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.")
725 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.")
726 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.")
727 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
728
729
730
731
732
733
734 - def stagePeer(self, targetDir, ownership=None, permissions=None):
735 """
736 Stages data from the peer into the indicated local target directory.
737
738 The target directory must already exist before this method is called. If
739 passed in, ownership and permissions will be applied to the files that
740 are copied.
741
742 @note: The returned count of copied files might be inaccurate if some of
743 the copied files already existed in the staging directory prior to the
744 copy taking place. We don't clear the staging directory first, because
745 some extension might also be using it.
746
747 @note: If you have user/group as strings, call the L{util.getUidGid} function
748 to get the associated uid/gid as an ownership tuple.
749
750 @note: Unlike the local peer version of this method, an I/O error might
751 or might not be raised if the directory is empty. Since we're using a
752 remote copy method, we just don't have the fine-grained control over our
753 exceptions that's available when we can look directly at the filesystem,
754 and we can't control whether the remote copy method thinks an empty
755 directory is an error.
756
757 @param targetDir: Target directory to write data into
758 @type targetDir: String representing a directory on disk
759
760 @param ownership: Owner and group that the staged files should have
761 @type ownership: Tuple of numeric ids C{(uid, gid)}
762
763 @param permissions: Permissions that the staged files should have
764 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
765
766 @return: Number of files copied from the source directory to the target directory.
767
768 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
769 @raise ValueError: If a path cannot be encoded properly.
770 @raise IOError: If there were no files to stage (i.e. the directory was empty)
771 @raise IOError: If there is an IO error copying a file.
772 @raise OSError: If there is an OS error copying or changing permissions on a file
773 """
774 targetDir = encodePath(targetDir)
775 if not os.path.isabs(targetDir):
776 logger.debug("Target directory [%s] not an absolute path." % targetDir)
777 raise ValueError("Target directory must be an absolute path.")
778 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
779 logger.debug("Target directory [%s] is not a directory or does not exist on disk." % targetDir)
780 raise ValueError("Target directory is not a directory or does not exist on disk.")
781 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name,
782 self._rcpCommand, self._rcpCommandList,
783 self.collectDir, targetDir,
784 ownership, permissions)
785 if count == 0:
786 raise IOError("Did not copy any files from local peer.")
787 return count
788
790 """
791 Checks the collect indicator in the peer's staging directory.
792
793 When a peer has completed collecting its backup files, it will write an
794 empty indicator file into its collect directory. This method checks to
795 see whether that indicator has been written. If the remote copy command
796 fails, we return C{False} as if the file weren't there.
797
798 If you need to, you can override the name of the collect indicator file
799 by passing in a different name.
800
801 @note: Apparently, we can't count on all rcp-compatible implementations
802 to return sensible errors for some error conditions. As an example, the
803 C{scp} command in Debian 'woody' returns a zero (normal) status even when
804 it can't find a host or if the login or path is invalid. Because of
805 this, the implementation of this method is rather convoluted.
806
807 @param collectIndicator: Name of the collect indicator file to check
808 @type collectIndicator: String representing name of a file in the collect directory
809
810 @return: Boolean true/false depending on whether the indicator exists.
811 @raise ValueError: If a path cannot be encoded properly.
812 """
813 try:
814 if collectIndicator is None:
815 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)
816 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR)
817 else:
818 collectIndicator = encodePath(collectIndicator)
819 sourceFile = os.path.join(self.collectDir, collectIndicator)
820 targetFile = os.path.join(self.workingDir, collectIndicator)
821 logger.debug("Fetch remote [%s] into [%s]." % (sourceFile, targetFile))
822 if os.path.exists(targetFile):
823 try:
824 os.remove(targetFile)
825 except:
826 raise Exception("Error: collect indicator [%s] already exists!" % targetFile)
827 try:
828 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name,
829 self._rcpCommand, self._rcpCommandList,
830 sourceFile, targetFile,
831 overwrite=False)
832 if os.path.exists(targetFile):
833 return True
834 else:
835 return False
836 except Exception, e:
837 logger.info("Failed looking for collect indicator: %s" % e)
838 return False
839 finally:
840 if os.path.exists(targetFile):
841 try:
842 os.remove(targetFile)
843 except: pass
844
846 """
847 Writes the stage indicator in the peer's staging directory.
848
849 When the master has completed collecting its backup files, it will write
850 an empty indicator file into the peer's collect directory. The presence
851 of this file implies that the staging process is complete.
852
853 If you need to, you can override the name of the stage indicator file by
854 passing in a different name.
855
856 @note: If you have user/group as strings, call the L{util.getUidGid} function
857 to get the associated uid/gid as an ownership tuple.
858
859 @param stageIndicator: Name of the indicator file to write
860 @type stageIndicator: String representing name of a file in the collect directory
861
862 @raise ValueError: If a path cannot be encoded properly.
863 @raise IOError: If there is an IO error creating the file.
864 @raise OSError: If there is an OS error creating or changing permissions on the file
865 """
866 stageIndicator = encodePath(stageIndicator)
867 if stageIndicator is None:
868 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
869 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
870 else:
871 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
872 targetFile = os.path.join(self.collectDir, stageIndicator)
873 try:
874 if not os.path.exists(sourceFile):
875 open(sourceFile, "w").write("")
876 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name,
877 self._rcpCommand, self._rcpCommandList,
878 sourceFile, targetFile)
879 finally:
880 if os.path.exists(sourceFile):
881 try:
882 os.remove(sourceFile)
883 except: pass
884
886 """
887 Executes a command on the peer via remote shell.
888
889 @param command: Command to execute
890 @type command: String command-line suitable for use with rsh.
891
892 @raise IOError: If there is an error executing the command on the remote peer.
893 """
894 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser,
895 self.name, self._rshCommand,
896 self._rshCommandList, command)
897
899 """
900 Executes a managed action on this peer.
901
902 @param action: Name of the action to execute.
903 @param fullBackup: Whether a full backup should be executed.
904
905 @raise IOError: If there is an error executing the action on the remote peer.
906 """
907 try:
908 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup)
909 self.executeRemoteCommand(command)
910 except IOError, e:
911 logger.info(e)
912 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))
913
914
915
916
917
918
919 @staticmethod
920 - def _getDirContents(path):
921 """
922 Returns the contents of a directory in terms of a Set.
923
924 The directory's contents are read as a L{FilesystemList} containing only
925 files, and then the list is converted into a set object for later use.
926
927 @param path: Directory path to get contents for
928 @type path: String representing a path on disk
929
930 @return: Set of files in the directory
931 @raise ValueError: If path is not a directory or does not exist.
932 """
933 contents = FilesystemList()
934 contents.excludeDirs = True
935 contents.excludeLinks = True
936 contents.addDirContents(path)
937 try:
938 return set(contents)
939 except:
940 import sets
941 return sets.Set(contents)
942
943 @staticmethod
944 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList,
945 sourceDir, targetDir, ownership=None, permissions=None):
946 """
947 Copies files from the source directory to the target directory.
948
949 This function is not recursive. Only the files in the directory will be
950 copied. Ownership and permissions will be left at their default values
951 if new values are not specified. Behavior when copying soft links from
952 the collect directory is dependent on the behavior of the specified rcp
953 command.
954
955 @note: The returned count of copied files might be inaccurate if some of
956 the copied files already existed in the staging directory prior to the
957 copy taking place. We don't clear the staging directory first, because
958 some extension might also be using it.
959
960 @note: If you have user/group as strings, call the L{util.getUidGid} function
961 to get the associated uid/gid as an ownership tuple.
962
963 @note: We don't have a good way of knowing exactly what files we copied
964 down from the remote peer, unless we want to parse the output of the rcp
965 command (ugh). We could change permissions on everything in the target
966 directory, but that's kind of ugly too. Instead, we use Python's set
967 functionality to figure out what files were added while we executed the
968 rcp command. This isn't perfect - for instance, it's not correct if
969 someone else is messing with the directory at the same time we're doing
970 the remote copy - but it's about as good as we're going to get.
971
972 @note: Apparently, we can't count on all rcp-compatible implementations
973 to return sensible errors for some error conditions. As an example, the
974 C{scp} command in Debian 'woody' returns a zero (normal) status even
975 when it can't find a host or if the login or path is invalid. We try
976 to work around this by issuing C{IOError} if we don't copy any files from
977 the remote host.
978
979 @param remoteUser: Name of the Cedar Backup user on the remote peer
980 @type remoteUser: String representing a username, valid via the copy command
981
982 @param localUser: Name of the Cedar Backup user on the current host
983 @type localUser: String representing a username, valid on the current host
984
985 @param remoteHost: Hostname of the remote peer
986 @type remoteHost: String representing a hostname, accessible via the copy command
987
988 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
989 @type rcpCommand: String representing a system command including required arguments
990
991 @param rcpCommandList: An rcp-compatible copy command to use for copying files
992 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
993
994 @param sourceDir: Source directory
995 @type sourceDir: String representing a directory on disk
996
997 @param targetDir: Target directory
998 @type targetDir: String representing a directory on disk
999
1000 @param ownership: Owner and group that the copied files should have
1001 @type ownership: Tuple of numeric ids C{(uid, gid)}
1002
1003 @param permissions: Permissions that the staged files should have
1004 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1005
1006 @return: Number of files copied from the source directory to the target directory.
1007
1008 @raise ValueError: If source or target is not a directory or does not exist.
1009 @raise IOError: If there is an IO error copying the files.
1010 """
1011 beforeSet = RemotePeer._getDirContents(targetDir)
1012 if localUser is not None:
1013 try:
1014 if not isRunningAsRoot():
1015 raise IOError("Only root can remote copy as another user.")
1016 except AttributeError: pass
1017 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir)
1018 command = resolveCommand(SU_COMMAND)
1019 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1020 if result != 0:
1021 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser))
1022 else:
1023 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir)
1024 command = resolveCommand(rcpCommandList)
1025 result = executeCommand(command, [copySource, targetDir])[0]
1026 if result != 0:
1027 raise IOError("Error (%d) copying files from remote host." % result)
1028 afterSet = RemotePeer._getDirContents(targetDir)
1029 if len(afterSet) == 0:
1030 raise IOError("Did not copy any files from remote peer.")
1031 differenceSet = afterSet.difference(beforeSet)
1032 if len(differenceSet) == 0:
1033 raise IOError("Apparently did not copy any new files from remote peer.")
1034 for targetFile in differenceSet:
1035 if ownership is not None:
1036 os.chown(targetFile, ownership[0], ownership[1])
1037 if permissions is not None:
1038 os.chmod(targetFile, permissions)
1039 return len(differenceSet)
1040
1041 @staticmethod
1042 - def _copyRemoteFile(remoteUser, localUser, remoteHost,
1043 rcpCommand, rcpCommandList,
1044 sourceFile, targetFile, ownership=None,
1045 permissions=None, overwrite=True):
1046 """
1047 Copies a remote source file to a target file.
1048
1049 @note: Internally, we have to go through and escape any spaces in the
1050 source path with double-backslash, otherwise things get screwed up. It
1051 doesn't seem to be required in the target path. I hope this is portable
1052 to various different rcp methods, but I guess it might not be (all I have
1053 to test with is OpenSSH).
1054
1055 @note: If you have user/group as strings, call the L{util.getUidGid} function
1056 to get the associated uid/gid as an ownership tuple.
1057
1058 @note: We will not overwrite a target file that exists when this method
1059 is invoked. If the target already exists, we'll raise an exception.
1060
1061 @note: Apparently, we can't count on all rcp-compatible implementations
1062 to return sensible errors for some error conditions. As an example, the
1063 C{scp} command in Debian 'woody' returns a zero (normal) status even when
1064 it can't find a host or if the login or path is invalid. We try to work
1065 around this by issuing C{IOError} the target file does not exist when
1066 we're done.
1067
1068 @param remoteUser: Name of the Cedar Backup user on the remote peer
1069 @type remoteUser: String representing a username, valid via the copy command
1070
1071 @param remoteHost: Hostname of the remote peer
1072 @type remoteHost: String representing a hostname, accessible via the copy command
1073
1074 @param localUser: Name of the Cedar Backup user on the current host
1075 @type localUser: String representing a username, valid on the current host
1076
1077 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1078 @type rcpCommand: String representing a system command including required arguments
1079
1080 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1081 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1082
1083 @param sourceFile: Source file to copy
1084 @type sourceFile: String representing a file on disk, as an absolute path
1085
1086 @param targetFile: Target file to create
1087 @type targetFile: String representing a file on disk, as an absolute path
1088
1089 @param ownership: Owner and group that the copied should have
1090 @type ownership: Tuple of numeric ids C{(uid, gid)}
1091
1092 @param permissions: Permissions that the staged files should have
1093 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1094
1095 @param overwrite: Indicates whether it's OK to overwrite the target file.
1096 @type overwrite: Boolean true/false.
1097
1098 @raise IOError: If the target file already exists.
1099 @raise IOError: If there is an IO error copying the file
1100 @raise OSError: If there is an OS error changing permissions on the file
1101 """
1102 if not overwrite:
1103 if os.path.exists(targetFile):
1104 raise IOError("Target file [%s] already exists." % targetFile)
1105 if localUser is not None:
1106 try:
1107 if not isRunningAsRoot():
1108 raise IOError("Only root can remote copy as another user.")
1109 except AttributeError: pass
1110 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile)
1111 command = resolveCommand(SU_COMMAND)
1112 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1113 if result != 0:
1114 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser))
1115 else:
1116 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ "))
1117 command = resolveCommand(rcpCommandList)
1118 result = executeCommand(command, [copySource, targetFile])[0]
1119 if result != 0:
1120 raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile))
1121 if not os.path.exists(targetFile):
1122 raise IOError("Apparently unable to copy file from remote host.")
1123 if ownership is not None:
1124 os.chown(targetFile, ownership[0], ownership[1])
1125 if permissions is not None:
1126 os.chmod(targetFile, permissions)
1127
1128 @staticmethod
1129 - def _pushLocalFile(remoteUser, localUser, remoteHost,
1130 rcpCommand, rcpCommandList,
1131 sourceFile, targetFile, overwrite=True):
1132 """
1133 Copies a local source file to a remote host.
1134
1135 @note: We will not overwrite a target file that exists when this method
1136 is invoked. If the target already exists, we'll raise an exception.
1137
1138 @note: Internally, we have to go through and escape any spaces in the
1139 source and target paths with double-backslash, otherwise things get
1140 screwed up. I hope this is portable to various different rcp methods,
1141 but I guess it might not be (all I have to test with is OpenSSH).
1142
1143 @note: If you have user/group as strings, call the L{util.getUidGid} function
1144 to get the associated uid/gid as an ownership tuple.
1145
1146 @param remoteUser: Name of the Cedar Backup user on the remote peer
1147 @type remoteUser: String representing a username, valid via the copy command
1148
1149 @param localUser: Name of the Cedar Backup user on the current host
1150 @type localUser: String representing a username, valid on the current host
1151
1152 @param remoteHost: Hostname of the remote peer
1153 @type remoteHost: String representing a hostname, accessible via the copy command
1154
1155 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1156 @type rcpCommand: String representing a system command including required arguments
1157
1158 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1159 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1160
1161 @param sourceFile: Source file to copy
1162 @type sourceFile: String representing a file on disk, as an absolute path
1163
1164 @param targetFile: Target file to create
1165 @type targetFile: String representing a file on disk, as an absolute path
1166
1167 @param overwrite: Indicates whether it's OK to overwrite the target file.
1168 @type overwrite: Boolean true/false.
1169
1170 @raise IOError: If there is an IO error copying the file
1171 @raise OSError: If there is an OS error changing permissions on the file
1172 """
1173 if not overwrite:
1174 if os.path.exists(targetFile):
1175 raise IOError("Target file [%s] already exists." % targetFile)
1176 if localUser is not None:
1177 try:
1178 if not isRunningAsRoot():
1179 raise IOError("Only root can remote copy as another user.")
1180 except AttributeError: pass
1181 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile)
1182 command = resolveCommand(SU_COMMAND)
1183 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1184 if result != 0:
1185 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser))
1186 else:
1187 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ "))
1188 command = resolveCommand(rcpCommandList)
1189 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0]
1190 if result != 0:
1191 raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile))
1192
1193 @staticmethod
1194 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):
1195 """
1196 Executes a command on the peer via remote shell.
1197
1198 @param remoteUser: Name of the Cedar Backup user on the remote peer
1199 @type remoteUser: String representing a username, valid on the remote host
1200
1201 @param localUser: Name of the Cedar Backup user on the current host
1202 @type localUser: String representing a username, valid on the current host
1203
1204 @param remoteHost: Hostname of the remote peer
1205 @type remoteHost: String representing a hostname, accessible via the copy command
1206
1207 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
1208 @type rshCommand: String representing a system command including required arguments
1209
1210 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer
1211 @type rshCommandList: Command as a list to be passed to L{util.executeCommand}
1212
1213 @param remoteCommand: The command to be executed on the remote host
1214 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.)
1215
1216 @raise IOError: If there is an error executing the remote command
1217 """
1218 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand)
1219 if localUser is not None:
1220 try:
1221 if not isRunningAsRoot():
1222 raise IOError("Only root can remote shell as another user.")
1223 except AttributeError: pass
1224 command = resolveCommand(SU_COMMAND)
1225 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1226 if result != 0:
1227 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand))
1228 else:
1229 command = resolveCommand(rshCommandList)
1230 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0]
1231 if result != 0:
1232 raise IOError("Command failed [%s]" % (actualCommand))
1233
1234 @staticmethod
1236 """
1237 Builds a Cedar Backup command line for the named action.
1238
1239 @note: If the cback command is None, then DEF_CBACK_COMMAND is used.
1240
1241 @param cbackCommand: cback command to execute, including required options
1242 @param action: Name of the action to execute.
1243 @param fullBackup: Whether a full backup should be executed.
1244
1245 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand.
1246 @raise ValueError: If action is None.
1247 """
1248 if action is None:
1249 raise ValueError("Action cannot be None.")
1250 if cbackCommand is None:
1251 cbackCommand = DEF_CBACK_COMMAND
1252 if fullBackup:
1253 return "%s --full %s" % (cbackCommand, action)
1254 else:
1255 return "%s %s" % (cbackCommand, action)
1256