1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to back up MySQL databases.
40
41 This is a Cedar Backup extension used to back up MySQL databases via the Cedar
42 Backup command line. It requires a new configuration section <mysql> and is
43 intended to be run either immediately before or immediately after the standard
44 collect action. Aside from its own configuration, it requires the options and
45 collect configuration sections in the standard Cedar Backup configuration file.
46
47 The backup is done via the C{mysqldump} command included with the MySQL
48 product. Output can be compressed using C{gzip} or C{bzip2}. Administrators
49 can configure the extension either to back up all databases or to back up only
50 specific databases. Note that this code always produces a full backup. There
51 is currently no facility for making incremental backups. If/when someone has a
52 need for this and can describe how to do it, I'll update this extension or
53 provide another.
54
55 The extension assumes that all configured databases can be backed up by a
56 single user. Often, the "root" database user will be used. An alternative is
57 to create a separate MySQL "backup" user and grant that user rights to read
58 (but not write) various databases as needed. This second option is probably
59 the best choice.
60
61 The extension accepts a username and password in configuration. However, you
62 probably do not want to provide those values in Cedar Backup configuration.
63 This is because Cedar Backup will provide these values to C{mysqldump} via the
64 command-line C{--user} and C{--password} switches, which will be visible to
65 other users in the process listing.
66
67 Instead, you should configure the username and password in one of MySQL's
68 configuration files. Typically, that would be done by putting a stanza like
69 this in C{/root/.my.cnf}::
70
71 [mysqldump]
72 user = root
73 password = <secret>
74
75 Regardless of whether you are using C{~/.my.cnf} or C{/etc/cback3.conf} to store
76 database login and password information, you should be careful about who is
77 allowed to view that information. Typically, this means locking down
78 permissions so that only the file owner can read the file contents (i.e. use
79 mode C{0600}).
80
81 @author: Kenneth J. Pronovici <pronovic@ieee.org>
82 """
83
84
85
86
87
88
89 import os
90 import logging
91 from gzip import GzipFile
92 from bz2 import BZ2File
93 from functools import total_ordering
94
95
96 from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
97 from CedarBackup3.xmlutil import readFirstChild, readString, readStringList, readBoolean
98 from CedarBackup3.config import VALID_COMPRESS_MODES
99 from CedarBackup3.util import resolveCommand, executeCommand
100 from CedarBackup3.util import ObjectTypeList, changeOwnership
101
102
103
104
105
106
107 logger = logging.getLogger("CedarBackup3.log.extend.mysql")
108 MYSQLDUMP_COMMAND = [ "mysqldump", ]
109
110
111
112
113
114
115 @total_ordering
116 -class MysqlConfig(object):
117
118 """
119 Class representing MySQL configuration.
120
121 The MySQL configuration information is used for backing up MySQL databases.
122
123 The following restrictions exist on data in this class:
124
125 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
126 - The 'all' flag must be 'Y' if no databases are defined.
127 - The 'all' flag must be 'N' if any databases are defined.
128 - Any values in the databases list must be strings.
129
130 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, user,
131 password, all, databases
132 """
133
134 - def __init__(self, user=None, password=None, compressMode=None, all=None, databases=None):
135 """
136 Constructor for the C{MysqlConfig} class.
137
138 @param user: User to execute backup as.
139 @param password: Password associated with user.
140 @param compressMode: Compress mode for backed-up files.
141 @param all: Indicates whether to back up all databases.
142 @param databases: List of databases to back up.
143 """
144 self._user = None
145 self._password = None
146 self._compressMode = None
147 self._all = None
148 self._databases = None
149 self.user = user
150 self.password = password
151 self.compressMode = compressMode
152 self.all = all
153 self.databases = databases
154
156 """
157 Official string representation for class instance.
158 """
159 return "MysqlConfig(%s, %s, %s, %s)" % (self.user, self.password, self.all, self.databases)
160
162 """
163 Informal string representation for class instance.
164 """
165 return self.__repr__()
166
168 """Equals operator, iplemented in terms of original Python 2 compare operator."""
169 return self.__cmp__(other) == 0
170
172 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
173 return self.__cmp__(other) < 0
174
176 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
177 return self.__cmp__(other) > 0
178
180 """
181 Original Python 2 comparison operator.
182 @param other: Other object to compare to.
183 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
184 """
185 if other is None:
186 return 1
187 if self.user != other.user:
188 if str(self.user or "") < str(other.user or ""):
189 return -1
190 else:
191 return 1
192 if self.password != other.password:
193 if str(self.password or "") < str(other.password or ""):
194 return -1
195 else:
196 return 1
197 if self.compressMode != other.compressMode:
198 if str(self.compressMode or "") < str(other.compressMode or ""):
199 return -1
200 else:
201 return 1
202 if self.all != other.all:
203 if self.all < other.all:
204 return -1
205 else:
206 return 1
207 if self.databases != other.databases:
208 if self.databases < other.databases:
209 return -1
210 else:
211 return 1
212 return 0
213
215 """
216 Property target used to set the user value.
217 """
218 if value is not None:
219 if len(value) < 1:
220 raise ValueError("User must be non-empty string.")
221 self._user = value
222
224 """
225 Property target used to get the user value.
226 """
227 return self._user
228
230 """
231 Property target used to set the password value.
232 """
233 if value is not None:
234 if len(value) < 1:
235 raise ValueError("Password must be non-empty string.")
236 self._password = value
237
239 """
240 Property target used to get the password value.
241 """
242 return self._password
243
245 """
246 Property target used to set the compress mode.
247 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
248 @raise ValueError: If the value is not valid.
249 """
250 if value is not None:
251 if value not in VALID_COMPRESS_MODES:
252 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
253 self._compressMode = value
254
256 """
257 Property target used to get the compress mode.
258 """
259 return self._compressMode
260
262 """
263 Property target used to set the 'all' flag.
264 No validations, but we normalize the value to C{True} or C{False}.
265 """
266 if value:
267 self._all = True
268 else:
269 self._all = False
270
272 """
273 Property target used to get the 'all' flag.
274 """
275 return self._all
276
278 """
279 Property target used to set the databases list.
280 Either the value must be C{None} or each element must be a string.
281 @raise ValueError: If the value is not a string.
282 """
283 if value is None:
284 self._databases = None
285 else:
286 for database in value:
287 if len(database) < 1:
288 raise ValueError("Each database must be a non-empty string.")
289 try:
290 saved = self._databases
291 self._databases = ObjectTypeList(str, "string")
292 self._databases.extend(value)
293 except Exception as e:
294 self._databases = saved
295 raise e
296
298 """
299 Property target used to get the databases list.
300 """
301 return self._databases
302
303 user = property(_getUser, _setUser, None, "User to execute backup as.")
304 password = property(_getPassword, _setPassword, None, "Password associated with user.")
305 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
306 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
307 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
308
309
310
311
312
313
314 @total_ordering
315 -class LocalConfig(object):
316
317 """
318 Class representing this extension's configuration document.
319
320 This is not a general-purpose configuration object like the main Cedar
321 Backup configuration object. Instead, it just knows how to parse and emit
322 MySQL-specific configuration values. Third parties who need to read and
323 write configuration related to this extension should access it through the
324 constructor, C{validate} and C{addConfig} methods.
325
326 @note: Lists within this class are "unordered" for equality comparisons.
327
328 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, mysql,
329 validate, addConfig
330 """
331
332 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
333 """
334 Initializes a configuration object.
335
336 If you initialize the object without passing either C{xmlData} or
337 C{xmlPath} then configuration will be empty and will be invalid until it
338 is filled in properly.
339
340 No reference to the original XML data or original path is saved off by
341 this class. Once the data has been parsed (successfully or not) this
342 original information is discarded.
343
344 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
345 method will be called (with its default arguments) against configuration
346 after successfully parsing any passed-in XML. Keep in mind that even if
347 C{validate} is C{False}, it might not be possible to parse the passed-in
348 XML document if lower-level validations fail.
349
350 @note: It is strongly suggested that the C{validate} option always be set
351 to C{True} (the default) unless there is a specific need to read in
352 invalid configuration from disk.
353
354 @param xmlData: XML data representing configuration.
355 @type xmlData: String data.
356
357 @param xmlPath: Path to an XML file on disk.
358 @type xmlPath: Absolute path to a file on disk.
359
360 @param validate: Validate the document after parsing it.
361 @type validate: Boolean true/false.
362
363 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
364 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
365 @raise ValueError: If the parsed configuration document is not valid.
366 """
367 self._mysql = None
368 self.mysql = None
369 if xmlData is not None and xmlPath is not None:
370 raise ValueError("Use either xmlData or xmlPath, but not both.")
371 if xmlData is not None:
372 self._parseXmlData(xmlData)
373 if validate:
374 self.validate()
375 elif xmlPath is not None:
376 with open(xmlPath) as f:
377 xmlData = f.read()
378 self._parseXmlData(xmlData)
379 if validate:
380 self.validate()
381
383 """
384 Official string representation for class instance.
385 """
386 return "LocalConfig(%s)" % (self.mysql)
387
389 """
390 Informal string representation for class instance.
391 """
392 return self.__repr__()
393
395 """Equals operator, iplemented in terms of original Python 2 compare operator."""
396 return self.__cmp__(other) == 0
397
399 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
400 return self.__cmp__(other) < 0
401
403 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
404 return self.__cmp__(other) > 0
405
407 """
408 Original Python 2 comparison operator.
409 Lists within this class are "unordered" for equality comparisons.
410 @param other: Other object to compare to.
411 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
412 """
413 if other is None:
414 return 1
415 if self.mysql != other.mysql:
416 if self.mysql < other.mysql:
417 return -1
418 else:
419 return 1
420 return 0
421
423 """
424 Property target used to set the mysql configuration value.
425 If not C{None}, the value must be a C{MysqlConfig} object.
426 @raise ValueError: If the value is not a C{MysqlConfig}
427 """
428 if value is None:
429 self._mysql = None
430 else:
431 if not isinstance(value, MysqlConfig):
432 raise ValueError("Value must be a C{MysqlConfig} object.")
433 self._mysql = value
434
436 """
437 Property target used to get the mysql configuration value.
438 """
439 return self._mysql
440
441 mysql = property(_getMysql, _setMysql, None, "Mysql configuration in terms of a C{MysqlConfig} object.")
442
444 """
445 Validates configuration represented by the object.
446
447 The compress mode must be filled in. Then, if the 'all' flag I{is} set,
448 no databases are allowed, and if the 'all' flag is I{not} set, at least
449 one database is required.
450
451 @raise ValueError: If one of the validations fails.
452 """
453 if self.mysql is None:
454 raise ValueError("Mysql section is required.")
455 if self.mysql.compressMode is None:
456 raise ValueError("Compress mode value is required.")
457 if self.mysql.all:
458 if self.mysql.databases is not None and self.mysql.databases != []:
459 raise ValueError("Databases cannot be specified if 'all' flag is set.")
460 else:
461 if self.mysql.databases is None or len(self.mysql.databases) < 1:
462 raise ValueError("At least one MySQL database must be indicated if 'all' flag is not set.")
463
465 """
466 Adds a <mysql> configuration section as the next child of a parent.
467
468 Third parties should use this function to write configuration related to
469 this extension.
470
471 We add the following fields to the document::
472
473 user //cb_config/mysql/user
474 password //cb_config/mysql/password
475 compressMode //cb_config/mysql/compress_mode
476 all //cb_config/mysql/all
477
478 We also add groups of the following items, one list element per
479 item::
480
481 database //cb_config/mysql/database
482
483 @param xmlDom: DOM tree as from C{impl.createDocument()}.
484 @param parentNode: Parent that the section should be appended to.
485 """
486 if self.mysql is not None:
487 sectionNode = addContainerNode(xmlDom, parentNode, "mysql")
488 addStringNode(xmlDom, sectionNode, "user", self.mysql.user)
489 addStringNode(xmlDom, sectionNode, "password", self.mysql.password)
490 addStringNode(xmlDom, sectionNode, "compress_mode", self.mysql.compressMode)
491 addBooleanNode(xmlDom, sectionNode, "all", self.mysql.all)
492 if self.mysql.databases is not None:
493 for database in self.mysql.databases:
494 addStringNode(xmlDom, sectionNode, "database", database)
495
497 """
498 Internal method to parse an XML string into the object.
499
500 This method parses the XML document into a DOM tree (C{xmlDom}) and then
501 calls a static method to parse the mysql configuration section.
502
503 @param xmlData: XML data to be parsed
504 @type xmlData: String data
505
506 @raise ValueError: If the XML cannot be successfully parsed.
507 """
508 (xmlDom, parentNode) = createInputDom(xmlData)
509 self._mysql = LocalConfig._parseMysql(parentNode)
510
511 @staticmethod
513 """
514 Parses a mysql configuration section.
515
516 We read the following fields::
517
518 user //cb_config/mysql/user
519 password //cb_config/mysql/password
520 compressMode //cb_config/mysql/compress_mode
521 all //cb_config/mysql/all
522
523 We also read groups of the following item, one list element per
524 item::
525
526 databases //cb_config/mysql/database
527
528 @param parentNode: Parent node to search beneath.
529
530 @return: C{MysqlConfig} object or C{None} if the section does not exist.
531 @raise ValueError: If some filled-in value is invalid.
532 """
533 mysql = None
534 section = readFirstChild(parentNode, "mysql")
535 if section is not None:
536 mysql = MysqlConfig()
537 mysql.user = readString(section, "user")
538 mysql.password = readString(section, "password")
539 mysql.compressMode = readString(section, "compress_mode")
540 mysql.all = readBoolean(section, "all")
541 mysql.databases = readStringList(section, "database")
542 return mysql
543
544
545
546
547
548
549
550
551
552
553 -def executeAction(configPath, options, config):
554 """
555 Executes the MySQL backup action.
556
557 @param configPath: Path to configuration file on disk.
558 @type configPath: String representing a path on disk.
559
560 @param options: Program command-line options.
561 @type options: Options object.
562
563 @param config: Program configuration.
564 @type config: Config object.
565
566 @raise ValueError: Under many generic error conditions
567 @raise IOError: If a backup could not be written for some reason.
568 """
569 logger.debug("Executing MySQL extended action.")
570 if config.options is None or config.collect is None:
571 raise ValueError("Cedar Backup configuration is not properly filled in.")
572 local = LocalConfig(xmlPath=configPath)
573 if local.mysql.all:
574 logger.info("Backing up all databases.")
575 _backupDatabase(config.collect.targetDir, local.mysql.compressMode, local.mysql.user, local.mysql.password,
576 config.options.backupUser, config.options.backupGroup, None)
577 else:
578 logger.debug("Backing up %d individual databases.", len(local.mysql.databases))
579 for database in local.mysql.databases:
580 logger.info("Backing up database [%s].", database)
581 _backupDatabase(config.collect.targetDir, local.mysql.compressMode, local.mysql.user, local.mysql.password,
582 config.options.backupUser, config.options.backupGroup, database)
583 logger.info("Executed the MySQL extended action successfully.")
584
585 -def _backupDatabase(targetDir, compressMode, user, password, backupUser, backupGroup, database=None):
586 """
587 Backs up an individual MySQL database, or all databases.
588
589 This internal method wraps the public method and adds some functionality,
590 like figuring out a filename, etc.
591
592 @param targetDir: Directory into which backups should be written.
593 @param compressMode: Compress mode to be used for backed-up files.
594 @param user: User to use for connecting to the database (if any).
595 @param password: Password associated with user (if any).
596 @param backupUser: User to own resulting file.
597 @param backupGroup: Group to own resulting file.
598 @param database: Name of database, or C{None} for all databases.
599
600 @return: Name of the generated backup file.
601
602 @raise ValueError: If some value is missing or invalid.
603 @raise IOError: If there is a problem executing the MySQL dump.
604 """
605 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
606 with outputFile:
607 backupDatabase(user, password, outputFile, database)
608 if not os.path.exists(filename):
609 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
610 changeOwnership(filename, backupUser, backupGroup)
611
613 """
614 Opens the output file used for saving the MySQL dump.
615
616 The filename is either C{"mysqldump.txt"} or C{"mysqldump-<database>.txt"}. The
617 C{".bz2"} extension is added if C{compress} is C{True}.
618
619 @param targetDir: Target directory to write file in.
620 @param database: Name of the database (if any)
621 @param compressMode: Compress mode to be used for backed-up files.
622
623 @return: Tuple of (Output file object, filename), file opened in binary mode for use with executeCommand()
624 """
625 if database is None:
626 filename = os.path.join(targetDir, "mysqldump.txt")
627 else:
628 filename = os.path.join(targetDir, "mysqldump-%s.txt" % database)
629 if compressMode == "gzip":
630 filename = "%s.gz" % filename
631 outputFile = GzipFile(filename, "wb")
632 elif compressMode == "bzip2":
633 filename = "%s.bz2" % filename
634 outputFile = BZ2File(filename, "wb")
635 else:
636 outputFile = open(filename, "wb")
637 logger.debug("MySQL dump file will be [%s].", filename)
638 return (outputFile, filename)
639
640
641
642
643
644
645 -def backupDatabase(user, password, backupFile, database=None):
646 """
647 Backs up an individual MySQL database, or all databases.
648
649 This function backs up either a named local MySQL database or all local
650 MySQL databases, using the passed-in user and password (if provided) for
651 connectivity. This function call I{always} results a full backup. There is
652 no facility for incremental backups.
653
654 The backup data will be written into the passed-in backup file. Normally,
655 this would be an object as returned from C{open()}, but it is possible to
656 use something like a C{GzipFile} to write compressed output. The caller is
657 responsible for closing the passed-in backup file.
658
659 Often, the "root" database user will be used when backing up all databases.
660 An alternative is to create a separate MySQL "backup" user and grant that
661 user rights to read (but not write) all of the databases that will be backed
662 up.
663
664 This function accepts a username and password. However, you probably do not
665 want to pass those values in. This is because they will be provided to
666 C{mysqldump} via the command-line C{--user} and C{--password} switches,
667 which will be visible to other users in the process listing.
668
669 Instead, you should configure the username and password in one of MySQL's
670 configuration files. Typically, this would be done by putting a stanza like
671 this in C{/root/.my.cnf}, to provide C{mysqldump} with the root database
672 username and its password::
673
674 [mysqldump]
675 user = root
676 password = <secret>
677
678 If you are executing this function as some system user other than root, then
679 the C{.my.cnf} file would be placed in the home directory of that user. In
680 either case, make sure to set restrictive permissions (typically, mode
681 C{0600}) on C{.my.cnf} to make sure that other users cannot read the file.
682
683 @param user: User to use for connecting to the database (if any)
684 @type user: String representing MySQL username, or C{None}
685
686 @param password: Password associated with user (if any)
687 @type password: String representing MySQL password, or C{None}
688
689 @param backupFile: File use for writing backup.
690 @type backupFile: Python file object as from C{open()} or C{file()}.
691
692 @param database: Name of the database to be backed up.
693 @type database: String representing database name, or C{None} for all databases.
694
695 @raise ValueError: If some value is missing or invalid.
696 @raise IOError: If there is a problem executing the MySQL dump.
697 """
698 args = [ "-all", "--flush-logs", "--opt", ]
699 if user is not None:
700 logger.warn("Warning: MySQL username will be visible in process listing (consider using ~/.my.cnf).")
701 args.append("--user=%s" % user)
702 if password is not None:
703 logger.warn("Warning: MySQL password will be visible in process listing (consider using ~/.my.cnf).")
704 args.append("--password=%s" % password)
705 if database is None:
706 args.insert(0, "--all-databases")
707 else:
708 args.insert(0, "--databases")
709 args.append(database)
710 command = resolveCommand(MYSQLDUMP_COMMAND)
711 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
712 if result != 0:
713 if database is None:
714 raise IOError("Error [%d] executing MySQL database dump for all databases." % result)
715 else:
716 raise IOError("Error [%d] executing MySQL database dump for database [%s]." % (result, database))
717