1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Provides an extension to back up MySQL databases.
41
42 This is a Cedar Backup extension used to back up MySQL databases via the Cedar
43 Backup command line. It requires a new configuration section <mysql> and is
44 intended to be run either immediately before or immediately after the standard
45 collect action. Aside from its own configuration, it requires the options and
46 collect configuration sections in the standard Cedar Backup configuration file.
47
48 The backup is done via the C{mysqldump} command included with the MySQL
49 product. Output can be compressed using C{gzip} or C{bzip2}. Administrators
50 can configure the extension either to back up all databases or to back up only
51 specific databases. Note that this code always produces a full backup. There
52 is currently no facility for making incremental backups. If/when someone has a
53 need for this and can describe how to do it, I'll update this extension or
54 provide another.
55
56 The extension assumes that all configured databases can be backed up by a
57 single user. Often, the "root" database user will be used. An alternative is
58 to create a separate MySQL "backup" user and grant that user rights to read
59 (but not write) various databases as needed. This second option is probably
60 the best choice.
61
62 The extension accepts a username and password in configuration. However, you
63 probably do not want to provide those values in Cedar Backup configuration.
64 This is because Cedar Backup will provide these values to C{mysqldump} via the
65 command-line C{--user} and C{--password} switches, which will be visible to
66 other users in the process listing.
67
68 Instead, you should configure the username and password in one of MySQL's
69 configuration files. Typically, that would be done by putting a stanza like
70 this in C{/root/.my.cnf}::
71
72 [mysqldump]
73 user = root
74 password = <secret>
75
76 Regardless of whether you are using C{~/.my.cnf} or C{/etc/cback.conf} to store
77 database login and password information, you should be careful about who is
78 allowed to view that information. Typically, this means locking down
79 permissions so that only the file owner can read the file contents (i.e. use
80 mode C{0600}).
81
82 @author: Kenneth J. Pronovici <pronovic@ieee.org>
83 """
84
85
86
87
88
89
90 import os
91 import logging
92 from gzip import GzipFile
93 from bz2 import BZ2File
94
95
96 from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
97 from CedarBackup2.xmlutil import readFirstChild, readString, readStringList, readBoolean
98 from CedarBackup2.config import VALID_COMPRESS_MODES
99 from CedarBackup2.util import resolveCommand, executeCommand
100 from CedarBackup2.util import ObjectTypeList, changeOwnership
101
102
103
104
105
106
107 logger = logging.getLogger("CedarBackup2.log.extend.mysql")
108 MYSQLDUMP_COMMAND = [ "mysqldump", ]
116
117 """
118 Class representing MySQL configuration.
119
120 The MySQL configuration information is used for backing up MySQL databases.
121
122 The following restrictions exist on data in this class:
123
124 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
125 - The 'all' flag must be 'Y' if no databases are defined.
126 - The 'all' flag must be 'N' if any databases are defined.
127 - Any values in the databases list must be strings.
128
129 @sort: __init__, __repr__, __str__, __cmp__, user, password, all, databases
130 """
131
132 - def __init__(self, user=None, password=None, compressMode=None, all=None, databases=None):
133 """
134 Constructor for the C{MysqlConfig} class.
135
136 @param user: User to execute backup as.
137 @param password: Password associated with user.
138 @param compressMode: Compress mode for backed-up files.
139 @param all: Indicates whether to back up all databases.
140 @param databases: List of databases to back up.
141 """
142 self._user = None
143 self._password = None
144 self._compressMode = None
145 self._all = None
146 self._databases = None
147 self.user = user
148 self.password = password
149 self.compressMode = compressMode
150 self.all = all
151 self.databases = databases
152
154 """
155 Official string representation for class instance.
156 """
157 return "MysqlConfig(%s, %s, %s, %s)" % (self.user, self.password, self.all, self.databases)
158
160 """
161 Informal string representation for class instance.
162 """
163 return self.__repr__()
164
166 """
167 Definition of equals operator for this class.
168 @param other: Other object to compare to.
169 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
170 """
171 if other is None:
172 return 1
173 if self.user != other.user:
174 if self.user < other.user:
175 return -1
176 else:
177 return 1
178 if self.password != other.password:
179 if self.password < other.password:
180 return -1
181 else:
182 return 1
183 if self.compressMode != other.compressMode:
184 if self.compressMode < other.compressMode:
185 return -1
186 else:
187 return 1
188 if self.all != other.all:
189 if self.all < other.all:
190 return -1
191 else:
192 return 1
193 if self.databases != other.databases:
194 if self.databases < other.databases:
195 return -1
196 else:
197 return 1
198 return 0
199
201 """
202 Property target used to set the user value.
203 """
204 if value is not None:
205 if len(value) < 1:
206 raise ValueError("User must be non-empty string.")
207 self._user = value
208
210 """
211 Property target used to get the user value.
212 """
213 return self._user
214
216 """
217 Property target used to set the password value.
218 """
219 if value is not None:
220 if len(value) < 1:
221 raise ValueError("Password must be non-empty string.")
222 self._password = value
223
225 """
226 Property target used to get the password value.
227 """
228 return self._password
229
231 """
232 Property target used to set the compress mode.
233 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
234 @raise ValueError: If the value is not valid.
235 """
236 if value is not None:
237 if value not in VALID_COMPRESS_MODES:
238 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
239 self._compressMode = value
240
242 """
243 Property target used to get the compress mode.
244 """
245 return self._compressMode
246
248 """
249 Property target used to set the 'all' flag.
250 No validations, but we normalize the value to C{True} or C{False}.
251 """
252 if value:
253 self._all = True
254 else:
255 self._all = False
256
258 """
259 Property target used to get the 'all' flag.
260 """
261 return self._all
262
264 """
265 Property target used to set the databases list.
266 Either the value must be C{None} or each element must be a string.
267 @raise ValueError: If the value is not a string.
268 """
269 if value is None:
270 self._databases = None
271 else:
272 for database in value:
273 if len(database) < 1:
274 raise ValueError("Each database must be a non-empty string.")
275 try:
276 saved = self._databases
277 self._databases = ObjectTypeList(basestring, "string")
278 self._databases.extend(value)
279 except Exception, e:
280 self._databases = saved
281 raise e
282
284 """
285 Property target used to get the databases list.
286 """
287 return self._databases
288
289 user = property(_getUser, _setUser, None, "User to execute backup as.")
290 password = property(_getPassword, _setPassword, None, "Password associated with user.")
291 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
292 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
293 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
294
301
302 """
303 Class representing this extension's configuration document.
304
305 This is not a general-purpose configuration object like the main Cedar
306 Backup configuration object. Instead, it just knows how to parse and emit
307 MySQL-specific configuration values. Third parties who need to read and
308 write configuration related to this extension should access it through the
309 constructor, C{validate} and C{addConfig} methods.
310
311 @note: Lists within this class are "unordered" for equality comparisons.
312
313 @sort: __init__, __repr__, __str__, __cmp__, mysql, validate, addConfig
314 """
315
316 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
317 """
318 Initializes a configuration object.
319
320 If you initialize the object without passing either C{xmlData} or
321 C{xmlPath} then configuration will be empty and will be invalid until it
322 is filled in properly.
323
324 No reference to the original XML data or original path is saved off by
325 this class. Once the data has been parsed (successfully or not) this
326 original information is discarded.
327
328 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
329 method will be called (with its default arguments) against configuration
330 after successfully parsing any passed-in XML. Keep in mind that even if
331 C{validate} is C{False}, it might not be possible to parse the passed-in
332 XML document if lower-level validations fail.
333
334 @note: It is strongly suggested that the C{validate} option always be set
335 to C{True} (the default) unless there is a specific need to read in
336 invalid configuration from disk.
337
338 @param xmlData: XML data representing configuration.
339 @type xmlData: String data.
340
341 @param xmlPath: Path to an XML file on disk.
342 @type xmlPath: Absolute path to a file on disk.
343
344 @param validate: Validate the document after parsing it.
345 @type validate: Boolean true/false.
346
347 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
348 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
349 @raise ValueError: If the parsed configuration document is not valid.
350 """
351 self._mysql = None
352 self.mysql = None
353 if xmlData is not None and xmlPath is not None:
354 raise ValueError("Use either xmlData or xmlPath, but not both.")
355 if xmlData is not None:
356 self._parseXmlData(xmlData)
357 if validate:
358 self.validate()
359 elif xmlPath is not None:
360 xmlData = open(xmlPath).read()
361 self._parseXmlData(xmlData)
362 if validate:
363 self.validate()
364
366 """
367 Official string representation for class instance.
368 """
369 return "LocalConfig(%s)" % (self.mysql)
370
372 """
373 Informal string representation for class instance.
374 """
375 return self.__repr__()
376
378 """
379 Definition of equals operator for this class.
380 Lists within this class are "unordered" for equality comparisons.
381 @param other: Other object to compare to.
382 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
383 """
384 if other is None:
385 return 1
386 if self.mysql != other.mysql:
387 if self.mysql < other.mysql:
388 return -1
389 else:
390 return 1
391 return 0
392
394 """
395 Property target used to set the mysql configuration value.
396 If not C{None}, the value must be a C{MysqlConfig} object.
397 @raise ValueError: If the value is not a C{MysqlConfig}
398 """
399 if value is None:
400 self._mysql = None
401 else:
402 if not isinstance(value, MysqlConfig):
403 raise ValueError("Value must be a C{MysqlConfig} object.")
404 self._mysql = value
405
407 """
408 Property target used to get the mysql configuration value.
409 """
410 return self._mysql
411
412 mysql = property(_getMysql, _setMysql, None, "Mysql configuration in terms of a C{MysqlConfig} object.")
413
415 """
416 Validates configuration represented by the object.
417
418 The compress mode must be filled in. Then, if the 'all' flag I{is} set,
419 no databases are allowed, and if the 'all' flag is I{not} set, at least
420 one database is required.
421
422 @raise ValueError: If one of the validations fails.
423 """
424 if self.mysql is None:
425 raise ValueError("Mysql section is required.")
426 if self.mysql.compressMode is None:
427 raise ValueError("Compress mode value is required.")
428 if self.mysql.all:
429 if self.mysql.databases is not None and self.mysql.databases != []:
430 raise ValueError("Databases cannot be specified if 'all' flag is set.")
431 else:
432 if self.mysql.databases is None or len(self.mysql.databases) < 1:
433 raise ValueError("At least one MySQL database must be indicated if 'all' flag is not set.")
434
436 """
437 Adds a <mysql> configuration section as the next child of a parent.
438
439 Third parties should use this function to write configuration related to
440 this extension.
441
442 We add the following fields to the document::
443
444 user //cb_config/mysql/user
445 password //cb_config/mysql/password
446 compressMode //cb_config/mysql/compress_mode
447 all //cb_config/mysql/all
448
449 We also add groups of the following items, one list element per
450 item::
451
452 database //cb_config/mysql/database
453
454 @param xmlDom: DOM tree as from C{impl.createDocument()}.
455 @param parentNode: Parent that the section should be appended to.
456 """
457 if self.mysql is not None:
458 sectionNode = addContainerNode(xmlDom, parentNode, "mysql")
459 addStringNode(xmlDom, sectionNode, "user", self.mysql.user)
460 addStringNode(xmlDom, sectionNode, "password", self.mysql.password)
461 addStringNode(xmlDom, sectionNode, "compress_mode", self.mysql.compressMode)
462 addBooleanNode(xmlDom, sectionNode, "all", self.mysql.all)
463 if self.mysql.databases is not None:
464 for database in self.mysql.databases:
465 addStringNode(xmlDom, sectionNode, "database", database)
466
468 """
469 Internal method to parse an XML string into the object.
470
471 This method parses the XML document into a DOM tree (C{xmlDom}) and then
472 calls a static method to parse the mysql configuration section.
473
474 @param xmlData: XML data to be parsed
475 @type xmlData: String data
476
477 @raise ValueError: If the XML cannot be successfully parsed.
478 """
479 (xmlDom, parentNode) = createInputDom(xmlData)
480 self._mysql = LocalConfig._parseMysql(parentNode)
481
482 @staticmethod
484 """
485 Parses a mysql configuration section.
486
487 We read the following fields::
488
489 user //cb_config/mysql/user
490 password //cb_config/mysql/password
491 compressMode //cb_config/mysql/compress_mode
492 all //cb_config/mysql/all
493
494 We also read groups of the following item, one list element per
495 item::
496
497 databases //cb_config/mysql/database
498
499 @param parentNode: Parent node to search beneath.
500
501 @return: C{MysqlConfig} object or C{None} if the section does not exist.
502 @raise ValueError: If some filled-in value is invalid.
503 """
504 mysql = None
505 section = readFirstChild(parentNode, "mysql")
506 if section is not None:
507 mysql = MysqlConfig()
508 mysql.user = readString(section, "user")
509 mysql.password = readString(section, "password")
510 mysql.compressMode = readString(section, "compress_mode")
511 mysql.all = readBoolean(section, "all")
512 mysql.databases = readStringList(section, "database")
513 return mysql
514
515
516
517
518
519
520
521
522
523
524 -def executeAction(configPath, options, config):
525 """
526 Executes the MySQL backup action.
527
528 @param configPath: Path to configuration file on disk.
529 @type configPath: String representing a path on disk.
530
531 @param options: Program command-line options.
532 @type options: Options object.
533
534 @param config: Program configuration.
535 @type config: Config object.
536
537 @raise ValueError: Under many generic error conditions
538 @raise IOError: If a backup could not be written for some reason.
539 """
540 logger.debug("Executing MySQL extended action.")
541 if config.options is None or config.collect is None:
542 raise ValueError("Cedar Backup configuration is not properly filled in.")
543 local = LocalConfig(xmlPath=configPath)
544 if local.mysql.all:
545 logger.info("Backing up all databases.")
546 _backupDatabase(config.collect.targetDir, local.mysql.compressMode, local.mysql.user, local.mysql.password,
547 config.options.backupUser, config.options.backupGroup, None)
548 else:
549 logger.debug("Backing up %d individual databases." % len(local.mysql.databases))
550 for database in local.mysql.databases:
551 logger.info("Backing up database [%s]." % database)
552 _backupDatabase(config.collect.targetDir, local.mysql.compressMode, local.mysql.user, local.mysql.password,
553 config.options.backupUser, config.options.backupGroup, database)
554 logger.info("Executed the MySQL extended action successfully.")
555
556 -def _backupDatabase(targetDir, compressMode, user, password, backupUser, backupGroup, database=None):
557 """
558 Backs up an individual MySQL database, or all databases.
559
560 This internal method wraps the public method and adds some functionality,
561 like figuring out a filename, etc.
562
563 @param targetDir: Directory into which backups should be written.
564 @param compressMode: Compress mode to be used for backed-up files.
565 @param user: User to use for connecting to the database (if any).
566 @param password: Password associated with user (if any).
567 @param backupUser: User to own resulting file.
568 @param backupGroup: Group to own resulting file.
569 @param database: Name of database, or C{None} for all databases.
570
571 @return: Name of the generated backup file.
572
573 @raise ValueError: If some value is missing or invalid.
574 @raise IOError: If there is a problem executing the MySQL dump.
575 """
576 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
577 try:
578 backupDatabase(user, password, outputFile, database)
579 finally:
580 outputFile.close()
581 if not os.path.exists(filename):
582 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
583 changeOwnership(filename, backupUser, backupGroup)
584
586 """
587 Opens the output file used for saving the MySQL dump.
588
589 The filename is either C{"mysqldump.txt"} or C{"mysqldump-<database>.txt"}. The
590 C{".bz2"} extension is added if C{compress} is C{True}.
591
592 @param targetDir: Target directory to write file in.
593 @param database: Name of the database (if any)
594 @param compressMode: Compress mode to be used for backed-up files.
595
596 @return: Tuple of (Output file object, filename)
597 """
598 if database is None:
599 filename = os.path.join(targetDir, "mysqldump.txt")
600 else:
601 filename = os.path.join(targetDir, "mysqldump-%s.txt" % database)
602 if compressMode == "gzip":
603 filename = "%s.gz" % filename
604 outputFile = GzipFile(filename, "w")
605 elif compressMode == "bzip2":
606 filename = "%s.bz2" % filename
607 outputFile = BZ2File(filename, "w")
608 else:
609 outputFile = open(filename, "w")
610 logger.debug("MySQL dump file will be [%s]." % filename)
611 return (outputFile, filename)
612
613
614
615
616
617
618 -def backupDatabase(user, password, backupFile, database=None):
619 """
620 Backs up an individual MySQL database, or all databases.
621
622 This function backs up either a named local MySQL database or all local
623 MySQL databases, using the passed-in user and password (if provided) for
624 connectivity. This function call I{always} results a full backup. There is
625 no facility for incremental backups.
626
627 The backup data will be written into the passed-in backup file. Normally,
628 this would be an object as returned from C{open()}, but it is possible to
629 use something like a C{GzipFile} to write compressed output. The caller is
630 responsible for closing the passed-in backup file.
631
632 Often, the "root" database user will be used when backing up all databases.
633 An alternative is to create a separate MySQL "backup" user and grant that
634 user rights to read (but not write) all of the databases that will be backed
635 up.
636
637 This function accepts a username and password. However, you probably do not
638 want to pass those values in. This is because they will be provided to
639 C{mysqldump} via the command-line C{--user} and C{--password} switches,
640 which will be visible to other users in the process listing.
641
642 Instead, you should configure the username and password in one of MySQL's
643 configuration files. Typically, this would be done by putting a stanza like
644 this in C{/root/.my.cnf}, to provide C{mysqldump} with the root database
645 username and its password::
646
647 [mysqldump]
648 user = root
649 password = <secret>
650
651 If you are executing this function as some system user other than root, then
652 the C{.my.cnf} file would be placed in the home directory of that user. In
653 either case, make sure to set restrictive permissions (typically, mode
654 C{0600}) on C{.my.cnf} to make sure that other users cannot read the file.
655
656 @param user: User to use for connecting to the database (if any)
657 @type user: String representing MySQL username, or C{None}
658
659 @param password: Password associated with user (if any)
660 @type password: String representing MySQL password, or C{None}
661
662 @param backupFile: File use for writing backup.
663 @type backupFile: Python file object as from C{open()} or C{file()}.
664
665 @param database: Name of the database to be backed up.
666 @type database: String representing database name, or C{None} for all databases.
667
668 @raise ValueError: If some value is missing or invalid.
669 @raise IOError: If there is a problem executing the MySQL dump.
670 """
671 args = [ "-all", "--flush-logs", "--opt", ]
672 if user is not None:
673 logger.warn("Warning: MySQL username will be visible in process listing (consider using ~/.my.cnf).")
674 args.append("--user=%s" % user)
675 if password is not None:
676 logger.warn("Warning: MySQL password will be visible in process listing (consider using ~/.my.cnf).")
677 args.append("--password=%s" % password)
678 if database is None:
679 args.insert(0, "--all-databases")
680 else:
681 args.insert(0, "--databases")
682 args.append(database)
683 command = resolveCommand(MYSQLDUMP_COMMAND)
684 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
685 if result != 0:
686 if database is None:
687 raise IOError("Error [%d] executing MySQL database dump for all databases." % result)
688 else:
689 raise IOError("Error [%d] executing MySQL database dump for database [%s]." % (result, database))
690