1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Synchonizes a local directory with an Amazon S3 bucket.
40
41 No configuration is required; all necessary information is taken from the
42 command-line. The only thing configuration would help with is the path
43 resolver interface, and it doesn't seem worth it to require configuration just
44 to get that.
45
46 @author: Kenneth J. Pronovici <pronovic@ieee.org>
47 """
48
49
50
51
52
53
54 import sys
55 import os
56 import logging
57 import getopt
58 import json
59 import chardet
60 import warnings
61
62
63 from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT
64 from CedarBackup2.filesystem import FilesystemList
65 from CedarBackup2.cli import setupLogging, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE
66 from CedarBackup2.util import Diagnostics, splitCommandLine, encodePath
67 from CedarBackup2.util import executeCommand
68
69
70
71
72
73
74 logger = logging.getLogger("CedarBackup2.log.tools.amazons3")
75
76 AWS_COMMAND = [ "aws" ]
77
78 SHORT_SWITCHES = "hVbql:o:m:OdsDvw"
79 LONG_SWITCHES = [ 'help', 'version', 'verbose', 'quiet',
80 'logfile=', 'owner=', 'mode=',
81 'output', 'debug', 'stack', 'diagnostics',
82 'verifyOnly', 'ignoreWarnings', ]
83
84
85
86
87
88
90
91
92
93
94
95 """
96 Class representing command-line options for the cback-amazons3-sync script.
97
98 The C{Options} class is a Python object representation of the command-line
99 options of the cback script.
100
101 The object representation is two-way: a command line string or a list of
102 command line arguments can be used to create an C{Options} object, and then
103 changes to the object can be propogated back to a list of command-line
104 arguments or to a command-line string. An C{Options} object can even be
105 created from scratch programmatically (if you have a need for that).
106
107 There are two main levels of validation in the C{Options} class. The first
108 is field-level validation. Field-level validation comes into play when a
109 given field in an object is assigned to or updated. We use Python's
110 C{property} functionality to enforce specific validations on field values,
111 and in some places we even use customized list classes to enforce
112 validations on list members. You should expect to catch a C{ValueError}
113 exception when making assignments to fields if you are programmatically
114 filling an object.
115
116 The second level of validation is post-completion validation. Certain
117 validations don't make sense until an object representation of options is
118 fully "complete". We don't want these validations to apply all of the time,
119 because it would make building up a valid object from scratch a real pain.
120 For instance, we might have to do things in the right order to keep from
121 throwing exceptions, etc.
122
123 All of these post-completion validations are encapsulated in the
124 L{Options.validate} method. This method can be called at any time by a
125 client, and will always be called immediately after creating a C{Options}
126 object from a command line and before exporting a C{Options} object back to
127 a command line. This way, we get acceptable ease-of-use but we also don't
128 accept or emit invalid command lines.
129
130 @note: Lists within this class are "unordered" for equality comparisons.
131
132 @sort: __init__, __repr__, __str__, __cmp__
133 """
134
135
136
137
138
139 - def __init__(self, argumentList=None, argumentString=None, validate=True):
140 """
141 Initializes an options object.
142
143 If you initialize the object without passing either C{argumentList} or
144 C{argumentString}, the object will be empty and will be invalid until it
145 is filled in properly.
146
147 No reference to the original arguments is saved off by this class. Once
148 the data has been parsed (successfully or not) this original information
149 is discarded.
150
151 The argument list is assumed to be a list of arguments, not including the
152 name of the command, something like C{sys.argv[1:]}. If you pass
153 C{sys.argv} instead, things are not going to work.
154
155 The argument string will be parsed into an argument list by the
156 L{util.splitCommandLine} function (see the documentation for that
157 function for some important notes about its limitations). There is an
158 assumption that the resulting list will be equivalent to C{sys.argv[1:]},
159 just like C{argumentList}.
160
161 Unless the C{validate} argument is C{False}, the L{Options.validate}
162 method will be called (with its default arguments) after successfully
163 parsing any passed-in command line. This validation ensures that
164 appropriate actions, etc. have been specified. Keep in mind that even if
165 C{validate} is C{False}, it might not be possible to parse the passed-in
166 command line, so an exception might still be raised.
167
168 @note: The command line format is specified by the L{_usage} function.
169 Call L{_usage} to see a usage statement for the cback script.
170
171 @note: It is strongly suggested that the C{validate} option always be set
172 to C{True} (the default) unless there is a specific need to read in
173 invalid command line arguments.
174
175 @param argumentList: Command line for a program.
176 @type argumentList: List of arguments, i.e. C{sys.argv}
177
178 @param argumentString: Command line for a program.
179 @type argumentString: String, i.e. "cback --verbose stage store"
180
181 @param validate: Validate the command line after parsing it.
182 @type validate: Boolean true/false.
183
184 @raise getopt.GetoptError: If the command-line arguments could not be parsed.
185 @raise ValueError: If the command-line arguments are invalid.
186 """
187 self._help = False
188 self._version = False
189 self._verbose = False
190 self._quiet = False
191 self._logfile = None
192 self._owner = None
193 self._mode = None
194 self._output = False
195 self._debug = False
196 self._stacktrace = False
197 self._diagnostics = False
198 self._verifyOnly = False
199 self._ignoreWarnings = False
200 self._sourceDir = None
201 self._s3BucketUrl = None
202 if argumentList is not None and argumentString is not None:
203 raise ValueError("Use either argumentList or argumentString, but not both.")
204 if argumentString is not None:
205 argumentList = splitCommandLine(argumentString)
206 if argumentList is not None:
207 self._parseArgumentList(argumentList)
208 if validate:
209 self.validate()
210
211
212
213
214
215
221
223 """
224 Informal string representation for class instance.
225 """
226 return self.__repr__()
227
228
229
230
231
232
318
319
320
321
322
323
325 """
326 Property target used to set the help flag.
327 No validations, but we normalize the value to C{True} or C{False}.
328 """
329 if value:
330 self._help = True
331 else:
332 self._help = False
333
335 """
336 Property target used to get the help flag.
337 """
338 return self._help
339
341 """
342 Property target used to set the version flag.
343 No validations, but we normalize the value to C{True} or C{False}.
344 """
345 if value:
346 self._version = True
347 else:
348 self._version = False
349
351 """
352 Property target used to get the version flag.
353 """
354 return self._version
355
357 """
358 Property target used to set the verbose flag.
359 No validations, but we normalize the value to C{True} or C{False}.
360 """
361 if value:
362 self._verbose = True
363 else:
364 self._verbose = False
365
367 """
368 Property target used to get the verbose flag.
369 """
370 return self._verbose
371
373 """
374 Property target used to set the quiet flag.
375 No validations, but we normalize the value to C{True} or C{False}.
376 """
377 if value:
378 self._quiet = True
379 else:
380 self._quiet = False
381
383 """
384 Property target used to get the quiet flag.
385 """
386 return self._quiet
387
389 """
390 Property target used to set the logfile parameter.
391 @raise ValueError: If the value cannot be encoded properly.
392 """
393 if value is not None:
394 if len(value) < 1:
395 raise ValueError("The logfile parameter must be a non-empty string.")
396 self._logfile = encodePath(value)
397
399 """
400 Property target used to get the logfile parameter.
401 """
402 return self._logfile
403
405 """
406 Property target used to set the owner parameter.
407 If not C{None}, the owner must be a C{(user,group)} tuple or list.
408 Strings (and inherited children of strings) are explicitly disallowed.
409 The value will be normalized to a tuple.
410 @raise ValueError: If the value is not valid.
411 """
412 if value is None:
413 self._owner = None
414 else:
415 if isinstance(value, str):
416 raise ValueError("Must specify user and group tuple for owner parameter.")
417 if len(value) != 2:
418 raise ValueError("Must specify user and group tuple for owner parameter.")
419 if len(value[0]) < 1 or len(value[1]) < 1:
420 raise ValueError("User and group tuple values must be non-empty strings.")
421 self._owner = (value[0], value[1])
422
424 """
425 Property target used to get the owner parameter.
426 The parameter is a tuple of C{(user, group)}.
427 """
428 return self._owner
429
431 """
432 Property target used to set the mode parameter.
433 """
434 if value is None:
435 self._mode = None
436 else:
437 try:
438 if isinstance(value, str):
439 value = int(value, 8)
440 else:
441 value = int(value)
442 except TypeError:
443 raise ValueError("Mode must be an octal integer >= 0, i.e. 644.")
444 if value < 0:
445 raise ValueError("Mode must be an octal integer >= 0. i.e. 644.")
446 self._mode = value
447
449 """
450 Property target used to get the mode parameter.
451 """
452 return self._mode
453
455 """
456 Property target used to set the output flag.
457 No validations, but we normalize the value to C{True} or C{False}.
458 """
459 if value:
460 self._output = True
461 else:
462 self._output = False
463
465 """
466 Property target used to get the output flag.
467 """
468 return self._output
469
471 """
472 Property target used to set the debug flag.
473 No validations, but we normalize the value to C{True} or C{False}.
474 """
475 if value:
476 self._debug = True
477 else:
478 self._debug = False
479
481 """
482 Property target used to get the debug flag.
483 """
484 return self._debug
485
487 """
488 Property target used to set the stacktrace flag.
489 No validations, but we normalize the value to C{True} or C{False}.
490 """
491 if value:
492 self._stacktrace = True
493 else:
494 self._stacktrace = False
495
497 """
498 Property target used to get the stacktrace flag.
499 """
500 return self._stacktrace
501
503 """
504 Property target used to set the diagnostics flag.
505 No validations, but we normalize the value to C{True} or C{False}.
506 """
507 if value:
508 self._diagnostics = True
509 else:
510 self._diagnostics = False
511
513 """
514 Property target used to get the diagnostics flag.
515 """
516 return self._diagnostics
517
519 """
520 Property target used to set the verifyOnly flag.
521 No validations, but we normalize the value to C{True} or C{False}.
522 """
523 if value:
524 self._verifyOnly = True
525 else:
526 self._verifyOnly = False
527
529 """
530 Property target used to get the verifyOnly flag.
531 """
532 return self._verifyOnly
533
535 """
536 Property target used to set the ignoreWarnings flag.
537 No validations, but we normalize the value to C{True} or C{False}.
538 """
539 if value:
540 self._ignoreWarnings = True
541 else:
542 self._ignoreWarnings = False
543
545 """
546 Property target used to get the ignoreWarnings flag.
547 """
548 return self._ignoreWarnings
549
551 """
552 Property target used to set the sourceDir parameter.
553 """
554 if value is not None:
555 if len(value) < 1:
556 raise ValueError("The sourceDir parameter must be a non-empty string.")
557 self._sourceDir = value
558
560 """
561 Property target used to get the sourceDir parameter.
562 """
563 return self._sourceDir
564
566 """
567 Property target used to set the s3BucketUrl parameter.
568 """
569 if value is not None:
570 if len(value) < 1:
571 raise ValueError("The s3BucketUrl parameter must be a non-empty string.")
572 self._s3BucketUrl = value
573
575 """
576 Property target used to get the s3BucketUrl parameter.
577 """
578 return self._s3BucketUrl
579
580 help = property(_getHelp, _setHelp, None, "Command-line help (C{-h,--help}) flag.")
581 version = property(_getVersion, _setVersion, None, "Command-line version (C{-V,--version}) flag.")
582 verbose = property(_getVerbose, _setVerbose, None, "Command-line verbose (C{-b,--verbose}) flag.")
583 quiet = property(_getQuiet, _setQuiet, None, "Command-line quiet (C{-q,--quiet}) flag.")
584 logfile = property(_getLogfile, _setLogfile, None, "Command-line logfile (C{-l,--logfile}) parameter.")
585 owner = property(_getOwner, _setOwner, None, "Command-line owner (C{-o,--owner}) parameter, as tuple C{(user,group)}.")
586 mode = property(_getMode, _setMode, None, "Command-line mode (C{-m,--mode}) parameter.")
587 output = property(_getOutput, _setOutput, None, "Command-line output (C{-O,--output}) flag.")
588 debug = property(_getDebug, _setDebug, None, "Command-line debug (C{-d,--debug}) flag.")
589 stacktrace = property(_getStacktrace, _setStacktrace, None, "Command-line stacktrace (C{-s,--stack}) flag.")
590 diagnostics = property(_getDiagnostics, _setDiagnostics, None, "Command-line diagnostics (C{-D,--diagnostics}) flag.")
591 verifyOnly = property(_getVerifyOnly, _setVerifyOnly, None, "Command-line verifyOnly (C{-v,--verifyOnly}) flag.")
592 ignoreWarnings = property(_getIgnoreWarnings, _setIgnoreWarnings, None, "Command-line ignoreWarnings (C{-w,--ignoreWarnings}) flag.")
593 sourceDir = property(_getSourceDir, _setSourceDir, None, "Command-line sourceDir, source of sync.")
594 s3BucketUrl = property(_getS3BucketUrl, _setS3BucketUrl, None, "Command-line s3BucketUrl, target of sync.")
595
596
597
598
599
600
602 """
603 Validates command-line options represented by the object.
604
605 Unless C{--help} or C{--version} are supplied, at least one action must
606 be specified. Other validations (as for allowed values for particular
607 options) will be taken care of at assignment time by the properties
608 functionality.
609
610 @note: The command line format is specified by the L{_usage} function.
611 Call L{_usage} to see a usage statement for the cback script.
612
613 @raise ValueError: If one of the validations fails.
614 """
615 if not self.help and not self.version and not self.diagnostics:
616 if self.sourceDir is None or self.s3BucketUrl is None:
617 raise ValueError("Source directory and S3 bucket URL are both required.")
618
620 """
621 Extracts options into a list of command line arguments.
622
623 The original order of the various arguments (if, indeed, the object was
624 initialized with a command-line) is not preserved in this generated
625 argument list. Besides that, the argument list is normalized to use the
626 long option names (i.e. --version rather than -V). The resulting list
627 will be suitable for passing back to the constructor in the
628 C{argumentList} parameter. Unlike L{buildArgumentString}, string
629 arguments are not quoted here, because there is no need for it.
630
631 Unless the C{validate} parameter is C{False}, the L{Options.validate}
632 method will be called (with its default arguments) against the
633 options before extracting the command line. If the options are not valid,
634 then an argument list will not be extracted.
635
636 @note: It is strongly suggested that the C{validate} option always be set
637 to C{True} (the default) unless there is a specific need to extract an
638 invalid command line.
639
640 @param validate: Validate the options before extracting the command line.
641 @type validate: Boolean true/false.
642
643 @return: List representation of command-line arguments.
644 @raise ValueError: If options within the object are invalid.
645 """
646 if validate:
647 self.validate()
648 argumentList = []
649 if self._help:
650 argumentList.append("--help")
651 if self.version:
652 argumentList.append("--version")
653 if self.verbose:
654 argumentList.append("--verbose")
655 if self.quiet:
656 argumentList.append("--quiet")
657 if self.logfile is not None:
658 argumentList.append("--logfile")
659 argumentList.append(self.logfile)
660 if self.owner is not None:
661 argumentList.append("--owner")
662 argumentList.append("%s:%s" % (self.owner[0], self.owner[1]))
663 if self.mode is not None:
664 argumentList.append("--mode")
665 argumentList.append("%o" % self.mode)
666 if self.output:
667 argumentList.append("--output")
668 if self.debug:
669 argumentList.append("--debug")
670 if self.stacktrace:
671 argumentList.append("--stack")
672 if self.diagnostics:
673 argumentList.append("--diagnostics")
674 if self.verifyOnly:
675 argumentList.append("--verifyOnly")
676 if self.ignoreWarnings:
677 argumentList.append("--ignoreWarnings")
678 if self.sourceDir is not None:
679 argumentList.append(self.sourceDir)
680 if self.s3BucketUrl is not None:
681 argumentList.append(self.s3BucketUrl)
682 return argumentList
683
685 """
686 Extracts options into a string of command-line arguments.
687
688 The original order of the various arguments (if, indeed, the object was
689 initialized with a command-line) is not preserved in this generated
690 argument string. Besides that, the argument string is normalized to use
691 the long option names (i.e. --version rather than -V) and to quote all
692 string arguments with double quotes (C{"}). The resulting string will be
693 suitable for passing back to the constructor in the C{argumentString}
694 parameter.
695
696 Unless the C{validate} parameter is C{False}, the L{Options.validate}
697 method will be called (with its default arguments) against the options
698 before extracting the command line. If the options are not valid, then
699 an argument string will not be extracted.
700
701 @note: It is strongly suggested that the C{validate} option always be set
702 to C{True} (the default) unless there is a specific need to extract an
703 invalid command line.
704
705 @param validate: Validate the options before extracting the command line.
706 @type validate: Boolean true/false.
707
708 @return: String representation of command-line arguments.
709 @raise ValueError: If options within the object are invalid.
710 """
711 if validate:
712 self.validate()
713 argumentString = ""
714 if self._help:
715 argumentString += "--help "
716 if self.version:
717 argumentString += "--version "
718 if self.verbose:
719 argumentString += "--verbose "
720 if self.quiet:
721 argumentString += "--quiet "
722 if self.logfile is not None:
723 argumentString += "--logfile \"%s\" " % self.logfile
724 if self.owner is not None:
725 argumentString += "--owner \"%s:%s\" " % (self.owner[0], self.owner[1])
726 if self.mode is not None:
727 argumentString += "--mode %o " % self.mode
728 if self.output:
729 argumentString += "--output "
730 if self.debug:
731 argumentString += "--debug "
732 if self.stacktrace:
733 argumentString += "--stack "
734 if self.diagnostics:
735 argumentString += "--diagnostics "
736 if self.verifyOnly:
737 argumentString += "--verifyOnly "
738 if self.ignoreWarnings:
739 argumentString += "--ignoreWarnings "
740 if self.sourceDir is not None:
741 argumentString += "\"%s\" " % self.sourceDir
742 if self.s3BucketUrl is not None:
743 argumentString += "\"%s\" " % self.s3BucketUrl
744 return argumentString
745
747 """
748 Internal method to parse a list of command-line arguments.
749
750 Most of the validation we do here has to do with whether the arguments
751 can be parsed and whether any values which exist are valid. We don't do
752 any validation as to whether required elements exist or whether elements
753 exist in the proper combination (instead, that's the job of the
754 L{validate} method).
755
756 For any of the options which supply parameters, if the option is
757 duplicated with long and short switches (i.e. C{-l} and a C{--logfile})
758 then the long switch is used. If the same option is duplicated with the
759 same switch (long or short), then the last entry on the command line is
760 used.
761
762 @param argumentList: List of arguments to a command.
763 @type argumentList: List of arguments to a command, i.e. C{sys.argv[1:]}
764
765 @raise ValueError: If the argument list cannot be successfully parsed.
766 """
767 switches = { }
768 opts, remaining = getopt.getopt(argumentList, SHORT_SWITCHES, LONG_SWITCHES)
769 for o, a in opts:
770 switches[o] = a
771 if switches.has_key("-h") or switches.has_key("--help"):
772 self.help = True
773 if switches.has_key("-V") or switches.has_key("--version"):
774 self.version = True
775 if switches.has_key("-b") or switches.has_key("--verbose"):
776 self.verbose = True
777 if switches.has_key("-q") or switches.has_key("--quiet"):
778 self.quiet = True
779 if switches.has_key("-l"):
780 self.logfile = switches["-l"]
781 if switches.has_key("--logfile"):
782 self.logfile = switches["--logfile"]
783 if switches.has_key("-o"):
784 self.owner = switches["-o"].split(":", 1)
785 if switches.has_key("--owner"):
786 self.owner = switches["--owner"].split(":", 1)
787 if switches.has_key("-m"):
788 self.mode = switches["-m"]
789 if switches.has_key("--mode"):
790 self.mode = switches["--mode"]
791 if switches.has_key("-O") or switches.has_key("--output"):
792 self.output = True
793 if switches.has_key("-d") or switches.has_key("--debug"):
794 self.debug = True
795 if switches.has_key("-s") or switches.has_key("--stack"):
796 self.stacktrace = True
797 if switches.has_key("-D") or switches.has_key("--diagnostics"):
798 self.diagnostics = True
799 if switches.has_key("-v") or switches.has_key("--verifyOnly"):
800 self.verifyOnly = True
801 if switches.has_key("-w") or switches.has_key("--ignoreWarnings"):
802 self.ignoreWarnings = True
803 try:
804 (self.sourceDir, self.s3BucketUrl) = remaining
805 except ValueError:
806 pass
807
808
809
810
811
812
813
814
815
816
818 """
819 Implements the command-line interface for the C{cback-amazons3-sync} script.
820
821 Essentially, this is the "main routine" for the cback-amazons3-sync script. It does
822 all of the argument processing for the script, and then also implements the
823 tool functionality.
824
825 This function looks pretty similiar to C{CedarBackup2.cli.cli()}. It's not
826 easy to refactor this code to make it reusable and also readable, so I've
827 decided to just live with the duplication.
828
829 A different error code is returned for each type of failure:
830
831 - C{1}: The Python interpreter version is < 2.7
832 - C{2}: Error processing command-line arguments
833 - C{3}: Error configuring logging
834 - C{5}: Backup was interrupted with a CTRL-C or similar
835 - C{6}: Error executing other parts of the script
836
837 @note: This script uses print rather than logging to the INFO level, because
838 it is interactive. Underlying Cedar Backup functionality uses the logging
839 mechanism exclusively.
840
841 @return: Error code as described above.
842 """
843 try:
844 if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 7]:
845 sys.stderr.write("Python 2 version 2.7 or greater required.\n")
846 return 1
847 except:
848
849 sys.stderr.write("Python 2 version 2.7 or greater required.\n")
850 return 1
851
852 try:
853 options = Options(argumentList=sys.argv[1:])
854 except Exception, e:
855 _usage()
856 sys.stderr.write(" *** Error: %s\n" % e)
857 return 2
858
859 if options.help:
860 _usage()
861 return 0
862 if options.version:
863 _version()
864 return 0
865 if options.diagnostics:
866 _diagnostics()
867 return 0
868
869 if options.stacktrace:
870 logfile = setupLogging(options)
871 else:
872 try:
873 logfile = setupLogging(options)
874 except Exception as e:
875 sys.stderr.write("Error setting up logging: %s\n" % e)
876 return 3
877
878 logger.info("Cedar Backup Amazon S3 sync run started.")
879 logger.info("Options were [%s]", options)
880 logger.info("Logfile is [%s]", logfile)
881 Diagnostics().logDiagnostics(method=logger.info)
882
883 if options.stacktrace:
884 _executeAction(options)
885 else:
886 try:
887 _executeAction(options)
888 except KeyboardInterrupt:
889 logger.error("Backup interrupted.")
890 logger.info("Cedar Backup Amazon S3 sync run completed with status 5.")
891 return 5
892 except Exception, e:
893 logger.error("Error executing backup: %s", e)
894 logger.info("Cedar Backup Amazon S3 sync run completed with status 6.")
895 return 6
896
897 logger.info("Cedar Backup Amazon S3 sync run completed with status 0.")
898 return 0
899
900
901
902
903
904
905
906
907
908
910 """
911 Prints usage information for the cback-amazons3-sync script.
912 @param fd: File descriptor used to print information.
913 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
914 """
915 fd.write("\n")
916 fd.write(" Usage: cback-amazons3-sync [switches] sourceDir s3bucketUrl\n")
917 fd.write("\n")
918 fd.write(" Cedar Backup Amazon S3 sync tool.\n")
919 fd.write("\n")
920 fd.write(" This Cedar Backup utility synchronizes a local directory to an Amazon S3\n")
921 fd.write(" bucket. After the sync is complete, a validation step is taken. An\n")
922 fd.write(" error is reported if the contents of the bucket do not match the\n")
923 fd.write(" source directory, or if the indicated size for any file differs.\n")
924 fd.write(" This tool is a wrapper over the AWS CLI command-line tool.\n")
925 fd.write("\n")
926 fd.write(" The following arguments are required:\n")
927 fd.write("\n")
928 fd.write(" sourceDir The local source directory on disk (must exist)\n")
929 fd.write(" s3BucketUrl The URL to the target Amazon S3 bucket\n")
930 fd.write("\n")
931 fd.write(" The following switches are accepted:\n")
932 fd.write("\n")
933 fd.write(" -h, --help Display this usage/help listing\n")
934 fd.write(" -V, --version Display version information\n")
935 fd.write(" -b, --verbose Print verbose output as well as logging to disk\n")
936 fd.write(" -q, --quiet Run quietly (display no output to the screen)\n")
937 fd.write(" -l, --logfile Path to logfile (default: %s)\n" % DEFAULT_LOGFILE)
938 fd.write(" -o, --owner Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1]))
939 fd.write(" -m, --mode Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE)
940 fd.write(" -O, --output Record some sub-command (i.e. aws) output to the log\n")
941 fd.write(" -d, --debug Write debugging information to the log (implies --output)\n")
942 fd.write(" -s, --stack Dump Python stack trace instead of swallowing exceptions\n")
943 fd.write(" -D, --diagnostics Print runtime diagnostics to the screen and exit\n")
944 fd.write(" -v, --verifyOnly Only verify the S3 bucket contents, do not make changes\n")
945 fd.write(" -w, --ignoreWarnings Ignore warnings about problematic filename encodings\n")
946 fd.write("\n")
947 fd.write(" Typical usage would be something like:\n")
948 fd.write("\n")
949 fd.write(" cback-amazons3-sync /home/myuser s3://example.com-backup/myuser\n")
950 fd.write("\n")
951 fd.write(" This will sync the contents of /home/myuser into the indicated bucket.\n")
952 fd.write("\n")
953
954
955
956
957
958
960 """
961 Prints version information for the cback script.
962 @param fd: File descriptor used to print information.
963 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
964 """
965 fd.write("\n")
966 fd.write(" Cedar Backup Amazon S3 sync tool.\n")
967 fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE))
968 fd.write("\n")
969 fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL))
970 fd.write(" See CREDITS for a list of included code and other contributors.\n")
971 fd.write(" This is free software; there is NO warranty. See the\n")
972 fd.write(" GNU General Public License version 2 for copying conditions.\n")
973 fd.write("\n")
974 fd.write(" Use the --help option for usage information.\n")
975 fd.write("\n")
976
977
978
979
980
981
983 """
984 Prints runtime diagnostics information.
985 @param fd: File descriptor used to print information.
986 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
987 """
988 fd.write("\n")
989 fd.write("Diagnostics:\n")
990 fd.write("\n")
991 Diagnostics().printDiagnostics(fd=fd, prefix=" ")
992 fd.write("\n")
993
994
995
996
997
998
1014
1015
1016
1017
1018
1019
1021 """
1022 Build a list of files in a source directory
1023 @param sourceDir: Local source directory
1024 @return: FilesystemList with contents of source directory
1025 """
1026 if not os.path.isdir(sourceDir):
1027 raise ValueError("Source directory does not exist on disk.")
1028 sourceFiles = FilesystemList()
1029 sourceFiles.addDirContents(sourceDir)
1030 return sourceFiles
1031
1032
1033
1034
1035
1036
1038 """
1039 Check source files, trying to guess which ones will have encoding problems.
1040 @param sourceDir: Local source directory
1041 @param sourceDir: Local source directory
1042 @raises ValueError: If a problem file is found
1043 @see U{http://opensourcehacker.com/2011/09/16/fix-linux-filename-encodings-with-python/}
1044 @see U{http://serverfault.com/questions/82821/how-to-tell-the-language-encoding-of-a-filename-on-linux}
1045 @see U{http://randysofia.com/2014/06/06/aws-cli-and-your-locale/}
1046 """
1047 with warnings.catch_warnings():
1048 warnings.simplefilter("ignore")
1049
1050 encoding = Diagnostics().encoding
1051
1052 failed = False
1053 for entry in sourceFiles:
1054 result = chardet.detect(entry)
1055 source = entry.decode(result["encoding"])
1056 try:
1057 target = source.encode(encoding)
1058 if source != target:
1059 logger.error("Inconsistent encoding for [%s]: got %s, but need %s", entry, result["encoding"], encoding)
1060 failed = True
1061 except UnicodeEncodeError:
1062 logger.error("Inconsistent encoding for [%s]: got %s, but need %s", entry, result["encoding"], encoding)
1063 failed = True
1064
1065 if not failed:
1066 logger.info("Completed checking source filename encoding (no problems found).")
1067 else:
1068 logger.error("Some filenames have inconsistent encodings and will likely cause sync problems.")
1069 logger.error("You may be able to fix this by setting a more sensible locale in your environment.")
1070 logger.error("Aternately, you can rename the problem files to be valid in the indicated locale.")
1071 logger.error("To ignore this warning and proceed anyway, use --ignoreWarnings")
1072 raise ValueError("Some filenames have inconsistent encodings and will likely cause sync problems.")
1073
1074
1075
1076
1077
1078
1080 """
1081 Synchronize a local directory to an Amazon S3 bucket.
1082 @param sourceDir: Local source directory
1083 @param s3BucketUrl: Target S3 bucket URL
1084 """
1085 logger.info("Synchronizing local source directory up to Amazon S3.")
1086 args = [ "s3", "sync", sourceDir, s3BucketUrl, "--delete", "--recursive", ]
1087 result = executeCommand(AWS_COMMAND, args, returnOutput=False)[0]
1088 if result != 0:
1089 raise IOError("Error [%d] calling AWS CLI synchronize bucket." % result)
1090
1091
1092
1093
1094
1095
1096 -def _verifyBucketContents(sourceDir, sourceFiles, s3BucketUrl):
1097 """
1098 Verify that a source directory is equivalent to an Amazon S3 bucket.
1099 @param sourceDir: Local source directory
1100 @param sourceFiles: Filesystem list containing contents of source directory
1101 @param s3BucketUrl: Target S3 bucket URL
1102 """
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125 (bucket, prefix) = s3BucketUrl.replace("s3://", "").split("/", 1)
1126
1127 query = "Contents[].{Key: Key, Size: Size}"
1128 args = [ "s3api", "list-objects", "--bucket", bucket, "--prefix", prefix, "--query", query, ]
1129 (result, data) = executeCommand(AWS_COMMAND, args, returnOutput=True)
1130 if result != 0:
1131 raise IOError("Error [%d] calling AWS CLI verify bucket contents." % result)
1132
1133 contents = { }
1134 for entry in json.loads("".join(data)):
1135 key = entry["Key"].replace(prefix, "")
1136 size = long(entry["Size"])
1137 contents[key] = size
1138
1139 failed = False
1140 for entry in sourceFiles:
1141 if os.path.isfile(entry):
1142 key = entry.replace(sourceDir, "")
1143 size = long(os.stat(entry).st_size)
1144 if not key in contents:
1145 logger.error("File was apparently not uploaded: [%s]", entry)
1146 failed = True
1147 else:
1148 if size != contents[key]:
1149 logger.error("File size differs [%s]: expected %s bytes but got %s bytes", entry, size, contents[key])
1150 failed = True
1151
1152 if not failed:
1153 logger.info("Completed verifying Amazon S3 bucket contents (no problems found).")
1154 else:
1155 logger.error("There were differences between source directory and target S3 bucket.")
1156 raise ValueError("There were differences between source directory and target S3 bucket.")
1157
1158
1159
1160
1161
1162
1163 if __name__ == "__main__":
1164 sys.exit(cli())
1165