1
2
3 raise ImportError('This module is deprecated. Use gmPG2.py.')
4
5
6
7
8
9 """Broker for PostgreSQL distributed backend connections.
10
11 @copyright: author
12
13 TODO: iterator/generator batch fetching:
14 - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad
15 - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4"
16
17 winner:
18 def resultset_functional_batchgenerator(cursor, size=100):
19 for results in iter(lambda: cursor.fetchmany(size), []):
20 for rec in results:
21 yield rec
22 """
23
24
25 __version__ = "$Revision: 1.90 $"
26 __author__ = "H.Herb <hherb@gnumed.net>, I.Haywood <i.haywood@ugrad.unimelb.edu.au>, K.Hilbert <Karsten.Hilbert@gmx.net>"
27 __license__ = 'GPL v2 or later (details at http://www.gnu.org)'
28
29 print "gmPG phased out, please replace with gmPG2"
30
31 import sys
32 sys.exit
33
34 _query_logging_verbosity = 1
35
36
37 assert(float(dbapi.apilevel) >= 2.0)
38 assert(dbapi.threadsafety > 0)
39 assert(dbapi.paramstyle == 'pyformat')
40
41 _listener_api = None
42
43
44 _default_client_encoding = {'wire': None, 'string': None}
45
46
47
48 if time.daylight:
49 tz = time.altzone
50 else:
51 tz = time.timezone
52
53
54 _default_client_timezone = "%+.1f" % (-tz / 3600.0)
55
56 _serialize_failure = "serialize access due to concurrent update"
57
58
59
60
61 QTablePrimaryKeyIndex = """
62 SELECT
63 indkey
64 FROM
65 pg_index
66 WHERE
67 indrelid =
68 (SELECT oid FROM pg_class WHERE relname = '%s');
69 """
70
71 query_pkey_name = """
72 SELECT
73 pga.attname
74 FROM
75 (pg_attribute pga inner join pg_index pgi on (pga.attrelid=pgi.indrelid))
76 WHERE
77 pga.attnum=pgi.indkey[0]
78 and
79 pgi.indisprimary is true
80 and
81 pga.attrelid=(SELECT oid FROM pg_class WHERE relname = %s)"""
82
83 query_fkey_names = """
84 select tgargs from pg_trigger where
85 tgname like 'RI%%'
86 and
87 tgrelid = (
88 select oid from pg_class where relname=%s
89 )
90 """
91
92
93 query_table_col_defs = """select
94 cols.column_name,
95 cols.udt_name
96 from
97 information_schema.columns cols
98 where
99 cols.table_schema = %s
100 and
101 cols.table_name = %s
102 order by
103 cols.ordinal_position"""
104
105 query_table_attributes = """select
106 cols.column_name
107 from
108 information_schema.columns cols
109 where
110 cols.table_schema = %s
111 and
112 cols.table_name = %s
113 order by
114 cols.ordinal_position"""
115
116 query_child_tables = """
117 select
118 pgn.nspname as namespace,
119 pgc.relname as table
120 from
121 pg_namespace pgn,
122 pg_class pgc
123 where
124 pgc.relnamespace = pgn.oid
125 and
126 pgc.oid in (
127 select inhrelid from pg_inherits where inhparent = (
128 select oid from pg_class where
129 relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and
130 relname = %(table)s
131 )
132 )"""
133
134
135
136 last_ro_cursor_desc = None
137
138
140 "maintains a static dictionary of available database connections"
141
142
143 __ro_conns = {}
144
145 __service2db_map = {}
146
147 __conn_use_count = {}
148
149 __is_connected = None
150
151 __listeners = {}
152
153 __login = None
154
155 - def __init__(self, login=None, encoding=None):
164
167
168
169
170
171
172
173 - def GetConnection(self, service="default", readonly=1, encoding=None, extra_verbose=None):
203
216
219
220 - def get_connection_for_user(self, user=None, password=None, service="default", encoding=None, extra_verbose=None):
221 """Get a connection for a given user.
222
223 This will return a connection just as GetConnection() would
224 except that the user to be used for authentication can be
225 specified. All the other parameters are going to be the
226 same, IOW it will connect to the same server, port and database
227 as any other connection obtained through this broker.
228
229 You will have to specify the password, of course, if it
230 is needed for PostgreSQL authentication.
231
232 This will always return a read-write connection.
233 """
234 if user is None:
235 _log.Log(gmLog.lErr, 'user must be given')
236 raise ValueError, 'gmPG.py::%s.get_connection_for_user(): user name must be given' % self.__class__.__name__
237
238 logininfo = self.GetLoginInfoFor(service)
239 logininfo.SetUser(user=user)
240 logininfo.SetPassword(passwd=password)
241
242 _log.Log(gmLog.lData, "requesting RW connection to service [%s]" % service)
243 conn = self.__pgconnect(logininfo, readonly = 0, encoding = encoding)
244 if conn is None:
245 return None
246
247 if extra_verbose:
248 conn.conn.toggleShowQuery
249
250 return conn
251
252
253
254 - def Listen(self, service, signal, callback):
255 """Listen to 'signal' from backend in an asynchronous thread.
256
257 If 'signal' is received from database 'service', activate
258 the 'callback' function"""
259
260
261
262 if _listener_api is None:
263 if not _import_listener_engine():
264 _log.Log(gmLog.lErr, 'cannot load backend listener code')
265 return None
266
267
268 try:
269 backend = ConnectionPool.__service2db_map[service]
270 except KeyError:
271 backend = 0
272 _log.Log(gmLog.lData, "connecting notification [%s] from service [%s] (id %s) with callback %s" % (signal, service, backend, callback))
273
274
275 if backend not in ConnectionPool.__listeners.keys():
276 auth = self.GetLoginInfoFor(service)
277 listener = _listener_api.BackendListener(
278 service,
279 auth.GetDatabase(),
280 auth.GetUser(),
281 auth.GetPassword(),
282 auth.GetHost(),
283 int(auth.GetPort())
284 )
285 ConnectionPool.__listeners[backend] = listener
286
287 listener = ConnectionPool.__listeners[backend]
288 listener.register_callback(signal, callback)
289 return 1
290
291 - def Unlisten(self, service, signal, callback):
302
304 try:
305 backend = self.__service2db_map[service]
306 except KeyError:
307 _log.Log(gmLog.lWarn, 'cannot stop listener on backend')
308 return None
309 try:
310 ConnectionPool.__listeners[backend].stop_thread()
311 del ConnectionPool.__listeners[backend]
312 except:
313 _log.LogException('cannot stop listener on backend [%s]' % backend, sys.exc_info(), verbose = 0)
314 return None
315 return 1
316
325
326
327
329 """list all distributed services available on this system
330 (according to configuration database)"""
331 return ConnectionPool.__ro_conns.keys()
332
334 """return login information for a particular service"""
335 if login is None:
336 dblogin = ConnectionPool.__login
337 else:
338 dblogin = copy.deepcopy(login)
339
340 try:
341 srvc_id = ConnectionPool.__service2db_map[service]
342 except KeyError:
343 return dblogin
344
345 if srvc_id == 0:
346 return dblogin
347
348
349 cfg_db = ConnectionPool.__ro_conns['default']
350 cursor = cfg_db.cursor()
351 cmd = "select name, host, port from cfg.db where pk=%s"
352 if not run_query(cursor, None, cmd, srvc_id):
353 _log.Log(gmLog.lPanic, 'cannot get login info for service [%s] with id [%s] from config database' % (service, srvc_id))
354 _log.Log(gmLog.lPanic, 'make sure your service-to-database mappings are properly configured')
355 _log.Log(gmLog.lWarn, 'trying to make do with default login parameters')
356 return dblogin
357 auth_data = cursor.fetchone()
358 idx = get_col_indices(cursor)
359 cursor.close()
360
361 try:
362 dblogin.SetDatabase(string.strip(auth_data[idx['name']]))
363 except: pass
364 try:
365 dblogin.SetHost(string.strip(auth_data[idx['host']]))
366 except: pass
367 try:
368 dblogin.SetPort(auth_data[idx['port']])
369 except: pass
370
371 return dblogin
372
373
374
376 """Initialize connections to all servers."""
377 if login is None and ConnectionPool.__is_connected is None:
378 try:
379 login = request_login_params()
380 except:
381 _log.LogException("Exception: Cannot connect to databases without login information !", sys.exc_info(), verbose=1)
382 raise gmExceptions.ConnectionError("Can't connect to database without login information!")
383
384 _log.Log(gmLog.lData, login.GetInfoStr())
385 ConnectionPool.__login = login
386
387
388 cfg_db = self.__pgconnect(login, readonly=1, encoding=encoding)
389 if cfg_db is None:
390 raise gmExceptions.ConnectionError, _('Cannot connect to configuration database with:\n\n[%s]') % login.GetInfoStr()
391
392
393 ConnectionPool.__ro_conns['default'] = cfg_db
394 cursor = cfg_db.cursor()
395
396 cursor.execute("select version()")
397 _log.Log(gmLog.lInfo, 'service [default/config] running on [%s]' % cursor.fetchone()[0])
398
399 cmd = "select name from cfg.distributed_db"
400 if not run_query(cursor, None, cmd):
401 cursor.close()
402 raise gmExceptions.ConnectionError("cannot load service names from configuration database")
403 services = cursor.fetchall()
404 for service in services:
405 ConnectionPool.__service2db_map[service[0]] = 0
406
407
408
409 cmd = "select * from cfg.config where profile=%s"
410 if not run_query(cursor, None, cmd, login.GetProfile()):
411 cursor.close()
412 raise gmExceptions.ConnectionError("cannot load user profile [%s] from database" % login.GetProfile())
413 databases = cursor.fetchall()
414 dbidx = get_col_indices(cursor)
415
416
417 for db in databases:
418
419 cursor.execute("select name from cfg.distributed_db where pk=%d" % db[dbidx['ddb']])
420 service = string.strip(cursor.fetchone()[0])
421
422 _log.Log(gmLog.lData, "mapping service [%s] to DB ID [%s]" % (service, db[dbidx['db']]))
423 ConnectionPool.__service2db_map[service] = db[dbidx['db']]
424
425 ConnectionPool.__conn_use_count[service] = 0
426 dblogin = self.GetLoginInfoFor(service, login)
427
428 conn = self.__pgconnect(dblogin, readonly=1, encoding=encoding)
429 if conn is None:
430 raise gmExceptions.ConnectionError, _('Cannot connect to database with:\n\n[%s]') % login.GetInfoStr()
431 ConnectionPool.__ro_conns[service] = conn
432
433 cursor.execute("select version()")
434 _log.Log(gmLog.lInfo, 'service [%s] running on [%s]' % (service, cursor.fetchone()[0]))
435 cursor.close()
436 ConnectionPool.__is_connected = 1
437 return ConnectionPool.__is_connected
438
439 - def __pgconnect(self, login, readonly=1, encoding=None):
440 """Connect to a postgres backend as specified by login object.
441
442 - returns a connection object
443 - encoding works like this:
444 - encoding specified in the call to __pgconnect() overrides
445 - encoding set by a call to gmPG.set_default_encoding() overrides
446 - encoding taken from Python string encoding
447 - wire_encoding and string_encoding must essentially just be different
448 names for one and the same (IOW entirely compatible) encodings, such
449 as "win1250" and "cp1250"
450 """
451 dsn = ""
452 hostport = ""
453 dsn = login.GetDBAPI_DSN()
454 hostport = "0"
455
456 if encoding is None:
457 encoding = _default_client_encoding
458
459
460
461
462 string_encoding = encoding['string']
463 if string_encoding is None:
464 string_encoding = _default_client_encoding['string']
465 if string_encoding is None:
466
467 string_encoding = locale.getlocale()[1]
468 _log.Log(gmLog.lWarn, 'client encoding not specified, this may lead to data corruption in some cases')
469 _log.Log(gmLog.lWarn, 'therefore the string encoding currently set in the active locale is used: [%s]' % string_encoding)
470 _log.Log(gmLog.lWarn, 'for this to have any chance to work the application MUST have called locale.setlocale() before')
471 _log.Log(gmLog.lInfo, 'using string encoding [%s] to encode Unicode strings for transmission to the database' % string_encoding)
472
473
474
475
476 wire_encoding = encoding['wire']
477 if wire_encoding is None:
478 wire_encoding = _default_client_encoding['wire']
479 if wire_encoding is None:
480 wire_encoding = string_encoding
481 if wire_encoding is None:
482 raise ValueError, '<wire_encoding> cannot be None'
483
484 try:
485
486 conn = dbapi.connect(dsn=dsn, client_encoding=(string_encoding, 'strict'), unicode_results=1)
487 except StandardError:
488 _log.LogException("database connection failed: DSN = [%s], host:port = [%s]" % (dsn, hostport), sys.exc_info(), verbose = 1)
489 return None
490
491
492 curs = conn.cursor()
493
494
495 cmd = "set client_encoding to '%s'" % wire_encoding
496 try:
497 curs.execute(cmd)
498 except:
499 curs.close()
500 conn.close()
501 _log.Log(gmLog.lErr, 'query [%s]' % cmd)
502 _log.LogException (
503 'cannot set string-on-the-wire client_encoding on connection to [%s], this would likely lead to data corruption' % wire_encoding,
504 sys.exc_info(),
505 verbose = _query_logging_verbosity
506 )
507 raise
508 _log.Log(gmLog.lData, 'string-on-the-wire client_encoding set to [%s]' % wire_encoding)
509
510
511
512 cmd = "set time zone '%s'" % _default_client_timezone
513 if not run_query(curs, None, cmd):
514 _log.Log(gmLog.lErr, 'cannot set client time zone to [%s]' % _default_client_timezone)
515 _log.Log(gmLog.lWarn, 'not setting this will lead to incorrect dates/times')
516 else:
517 _log.Log (gmLog.lData, 'time zone set to [%s]' % _default_client_timezone)
518
519
520
521 cmd = "set datestyle to 'ISO'"
522 if not run_query(curs, None, cmd):
523 _log.Log(gmLog.lErr, 'cannot set client date style to ISO')
524 _log.Log(gmLog.lWarn, 'you better use other means to make your server delivers valid ISO timestamps with time zone')
525
526
527 if readonly:
528 isolation_level = 'READ COMMITTED'
529 else:
530 isolation_level = 'SERIALIZABLE'
531 cmd = 'set session characteristics as transaction isolation level %s' % isolation_level
532 if not run_query(curs, None, cmd):
533 curs.close()
534 conn.close()
535 _log.Log(gmLog.lErr, 'cannot set connection characteristics to [%s]' % isolation_level)
536 return None
537
538
539 if readonly:
540 access_mode = 'READ ONLY'
541 else:
542 access_mode = 'READ WRITE'
543 _log.Log(gmLog.lData, "setting session to [%s] for %s@%s:%s" % (access_mode, login.GetUser(), login.GetHost(), login.GetDatabase()))
544 cmd = 'set session characteristics as transaction %s' % access_mode
545 if not run_query(curs, 0, cmd):
546 _log.Log(gmLog.lErr, 'cannot set connection characteristics to [%s]' % access_mode)
547 curs.close()
548 conn.close()
549 return None
550
551 conn.commit()
552 curs.close()
553 return conn
554
581
582
583
584
586 "returns the attribute names of the fetched rows in natural sequence as a list"
587 names=[]
588 for d in cursor.description:
589 names.append(d[0])
590 return names
591
592 -def run_query(aCursor=None, verbosity=None, aQuery=None, *args):
593
594 if aCursor is None:
595 _log.Log(gmLog.lErr, 'need cursor to run query')
596 return None
597 if aQuery is None:
598 _log.Log(gmLog.lErr, 'need query to run it')
599 return None
600 if verbosity is None:
601 verbosity = _query_logging_verbosity
602
603
604 try:
605 aCursor.execute(aQuery, *args)
606 except:
607 _log.LogException("query >>>%s<<< with args >>>%s<<< failed" % (aQuery, args), sys.exc_info(), verbose = verbosity)
608 return None
609
610
611 return 1
612
613 -def run_commit2(link_obj=None, queries=None, end_tx=False, max_tries=1, extra_verbose=False, get_col_idx = False):
614 """Convenience function for running a transaction
615 that is supposed to get committed.
616
617 <link_obj>
618 can be either:
619 - a cursor
620 - a connection
621 - a service name
622
623 <queries>
624 is a list of (query, [args]) tuples to be
625 executed as a single transaction, the last
626 query may usefully return rows (such as a
627 "select currval('some_sequence')" statement)
628
629 <end_tx>
630 - controls whether the transaction is finalized (eg.
631 committed/rolled back) or not, this allows the
632 call to run_commit2() to be part of a framing
633 transaction
634 - if <link_obj> is a service name the transaction is
635 always finalized regardless of what <end_tx> says
636 - if link_obj is a connection then <end_tx> will
637 default to False unless it is explicitly set to
638 True which is taken to mean "yes, you do have full
639 control over the transaction" in which case the
640 transaction is properly finalized
641
642 <max_tries>
643 - controls the number of times a transaction is retried
644 after a concurrency error
645 - note that *all* <queries> are rerun if a concurrency
646 error occurrs
647 - max_tries is honored if and only if link_obj is a service
648 name such that we have full control over the transaction
649
650 <get_col_idx>
651 - if true, the returned data will include a dictionary
652 mapping field names to column positions
653 - if false, the returned data returns an empty dict
654
655 method result:
656 - returns a tuple (status, data)
657 - <status>:
658 * True - if all queries succeeded (also if there were 0 queries)
659 * False - if *any* error occurred
660 - <data> if <status> is True:
661 * (None, {}) if last query did not return rows
662 * ("fetchall() result", <index>) if last query returned any rows
663 * for <index> see <get_col_idx>
664 - <data> if <status> is False:
665 * a tuple (error, message) where <error> can be:
666 * 1: unspecified error
667 * 2: concurrency error
668 * 3: constraint violation (non-primary key)
669 * 4: access violation
670 """
671
672 if queries is None:
673 return (False, (1, 'forgot to pass in queries'))
674 if len(queries) == 0:
675 return (True, 'no queries to execute')
676
677
678
679 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'):
680 return __commit2cursor(cursor=link_obj, queries=queries, extra_verbose=extra_verbose, get_col_idx=get_col_idx)
681
682 if (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')):
683 return __commit2conn(conn=link_obj, queries=queries, end_tx=end_tx, extra_verbose=extra_verbose, get_col_idx=get_col_idx)
684
685 return __commit2service(service=link_obj, queries=queries, max_tries=max_tries, extra_verbose=extra_verbose, get_col_idx=get_col_idx)
686
687 -def __commit2service(service=None, queries=None, max_tries=1, extra_verbose=False, get_col_idx=False):
688
689 try: int(max_tries)
690 except ValueEror: max_tries = 1
691 if max_tries > 4:
692 max_tries = 4
693 if max_tries < 1:
694 max_tries = 1
695
696 pool = ConnectionPool()
697 conn = pool.GetConnection(str(service), readonly = 0)
698 if conn is None:
699 msg = 'cannot connect to service [%s]'
700 _log.Log(gmLog.lErr, msg % service)
701 return (False, (1, _(msg) % service))
702 if extra_verbose:
703 conn.conn.toggleShowQuery
704 curs = conn.cursor()
705 for attempt in range(0, max_tries):
706 if extra_verbose:
707 _log.Log(gmLog.lData, 'attempt %s' % attempt)
708
709 for query, args in queries:
710 if extra_verbose:
711 t1 = time.time()
712 try:
713 curs.execute(query, *args)
714
715 except:
716 if extra_verbose:
717 duration = time.time() - t1
718 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration)
719 conn.rollback()
720 exc_info = sys.exc_info()
721 typ, val, tb = exc_info
722 if str(val).find(_serialize_failure) > 0:
723 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update')
724 if attempt < max_tries:
725
726 time.sleep(0.1)
727 continue
728 curs.close()
729 conn.close()
730 return (False, (2, 'l'))
731
732 _log.Log(gmLog.lErr, 'query: %s' % query[:2048])
733 try:
734 _log.Log(gmLog.lErr, 'argument: %s' % str(args)[:2048])
735 except MemoryError:
736 pass
737 _log.LogException("query failed on link [%s]" % service, exc_info)
738 if extra_verbose:
739 __log_PG_settings(curs)
740 curs.close()
741 conn.close()
742 tmp = str(val).replace('ERROR:', '')
743 tmp = tmp.replace('ExecAppend:', '')
744 tmp = tmp.strip()
745 return (False, (1, _('SQL: %s') % tmp))
746
747 if extra_verbose:
748 duration = time.time() - t1
749 _log.Log(gmLog.lData, 'query: %s' % query[:2048])
750 try:
751 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048])
752 except MemoryError:
753 pass
754 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % service)
755 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration))
756
757 break
758
759
760 data = None
761 idx = {}
762
763
764
765
766
767
768 try:
769 data = curs.fetchall()
770 except:
771 if extra_verbose:
772 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows')
773
774 if curs.description is not None:
775 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?')
776 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description))
777 conn.commit()
778 if get_col_idx:
779 idx = get_col_indices(curs)
780 curs.close()
781 conn.close()
782 return (True, (data, idx))
783
784 -def __commit2conn(conn=None, queries=None, end_tx=False, extra_verbose=False, get_col_idx=False):
785 if extra_verbose:
786 conn.conn.toggleShowQuery
787
788
789 curs = conn.cursor()
790
791
792 for query, args in queries:
793 if extra_verbose:
794 t1 = time.time()
795 try:
796 curs.execute(query, *args)
797 except:
798 if extra_verbose:
799 duration = time.time() - t1
800 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration)
801 conn.rollback()
802 exc_info = sys.exc_info()
803 typ, val, tb = exc_info
804 if str(val).find(_serialize_failure) > 0:
805 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update')
806 curs.close()
807 if extra_verbose:
808 conn.conn.toggleShowQuery
809 return (False, (2, 'l'))
810
811 _log.Log(gmLog.lErr, 'query: %s' % query[:2048])
812 try:
813 _log.Log(gmLog.lErr, 'args : %s' % str(args)[:2048])
814 except MemoryError:
815 pass
816 _log.LogException("query failed on link [%s]" % conn, exc_info)
817 if extra_verbose:
818 __log_PG_settings(curs)
819 curs.close()
820 tmp = str(val).replace('ERROR:', '')
821 tmp = tmp.replace('ExecAppend:', '')
822 tmp = tmp.strip()
823 if extra_verbose:
824 conn.conn.toggleShowQuery
825 return (False, (1, _('SQL: %s') % tmp))
826
827 if extra_verbose:
828 duration = time.time() - t1
829 _log.Log(gmLog.lData, 'query: %s' % query[:2048])
830 try:
831 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048])
832 except MemoryError:
833 pass
834 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % conn)
835 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration))
836
837 if extra_verbose:
838 conn.conn.toggleShowQuery
839
840 data = None
841 idx = {}
842
843
844
845
846
847
848 try:
849 data = curs.fetchall()
850 except:
851 if extra_verbose:
852 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows')
853
854 if curs.description is not None:
855 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?')
856 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description))
857 if end_tx:
858 conn.commit()
859 if get_col_idx:
860 idx = get_col_indices(curs)
861 curs.close()
862 return (True, (data, idx))
863
864 -def __commit2cursor(cursor=None, queries=None, extra_verbose=False, get_col_idx=False):
865
866 for query, args in queries:
867 if extra_verbose:
868 t1 = time.time()
869 try:
870 curs.execute(query, *args)
871 except:
872 if extra_verbose:
873 duration = time.time() - t1
874 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration)
875 exc_info = sys.exc_info()
876 typ, val, tb = exc_info
877 if str(val).find(_serialize_failure) > 0:
878 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update')
879 return (False, (2, 'l'))
880
881 _log.Log(gmLog.lErr, 'query: %s' % query[:2048])
882 try:
883 _log.Log(gmLog.lErr, 'args : %s' % str(args)[:2048])
884 except MemoryError:
885 pass
886 _log.LogException("query failed on link [%s]" % cursor, exc_info)
887 if extra_verbose:
888 __log_PG_settings(curs)
889 tmp = str(val).replace('ERROR:', '')
890 tmp = tmp.replace('ExecAppend:', '')
891 tmp = tmp.strip()
892 return (False, (1, _('SQL: %s') % tmp))
893
894 if extra_verbose:
895 duration = time.time() - t1
896 _log.Log(gmLog.lData, 'query: %s' % query[:2048])
897 try:
898 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048])
899 except MemoryError:
900 pass
901 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % cursor)
902 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration))
903
904
905 data = None
906 idx = {}
907
908
909
910
911
912
913 try:
914 data = curs.fetchall()
915 except:
916 if extra_verbose:
917 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows')
918
919 if curs.description is not None:
920 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?')
921 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description))
922 if get_col_idx:
923 idx = get_col_indices(curs)
924 return (True, (data, idx))
925
926 -def run_commit(link_obj = None, queries = None, return_err_msg = None):
927 """Convenience function for running a transaction
928 that is supposed to get committed.
929
930 - link_obj can be
931 - a cursor: rollback/commit must be done by the caller
932 - a connection: rollback/commit is handled
933 - a service name: rollback/commit is handled
934
935 - queries is a list of (query, [args]) tuples
936 - executed as a single transaction
937
938 - returns:
939 - a tuple (<value>, error) if return_err_msg is True
940 - a scalar <value> if return_err_msg is False
941
942 - <value> will be
943 - None: if any query failed
944 - 1: if all queries succeeded (also 0 queries)
945 - data: if the last query returned rows
946 """
947 print "DEPRECATION WARNING: gmPG.run_commit() is deprecated, use run_commit2() instead"
948
949
950 if link_obj is None:
951 raise TypeError, 'gmPG.run_commit(): link_obj must be of type service name, connection or cursor'
952 if queries is None:
953 raise TypeError, 'gmPG.run_commit(): forgot to pass in queries'
954 if len(queries) == 0:
955 _log.Log(gmLog.lWarn, 'no queries to execute ?!?')
956 if return_err_msg:
957 return (1, 'no queries to execute ?!?')
958 return 1
959
960 close_cursor = noop
961 close_conn = noop
962 commit = noop
963 rollback = noop
964
965 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'):
966 curs = link_obj
967
968 elif (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')):
969 curs = link_obj.cursor()
970 close_cursor = curs.close
971 conn = link_obj
972 commit = link_obj.commit
973 rollback = link_obj.rollback
974
975 else:
976 pool = ConnectionPool()
977 conn = pool.GetConnection(link_obj, readonly = 0)
978 if conn is None:
979 _log.Log(gmLog.lErr, 'cannot connect to service [%s]' % link_obj)
980 if return_err_msg:
981 return (None, _('cannot connect to service [%s]') % link_obj)
982 return None
983 curs = conn.cursor()
984 close_cursor = curs.close
985 close_conn = conn.close
986 commit = conn.commit
987 rollback = conn.rollback
988
989 for query, args in queries:
990
991 try:
992 curs.execute (query, *args)
993 except:
994 rollback()
995 exc_info = sys.exc_info()
996 _log.LogException ("RW query >>>%s<<< with args >>>%s<<< failed on link [%s]" % (query[:1024], str(args)[:1024], link_obj), exc_info, verbose = _query_logging_verbosity)
997 __log_PG_settings(curs)
998 close_cursor()
999 close_conn()
1000 if return_err_msg:
1001 typ, val, tb = exc_info
1002 tmp = string.replace(str(val), 'ERROR:', '')
1003 tmp = string.replace(tmp, 'ExecAppend:', '')
1004 tmp = string.strip(tmp)
1005 return (None, 'SQL: %s' % tmp)
1006 return None
1007
1008
1009 if _query_logging_verbosity == 1:
1010 _log.Log(gmLog.lData, '%s rows affected by >>>%s<<<' % (curs.rowcount, query))
1011
1012 data = None
1013
1014
1015
1016
1017
1018 try:
1019 data = curs.fetchall()
1020 if _query_logging_verbosity == 1:
1021 _log.Log(gmLog.lData, 'last query returned %s rows' % curs.rowcount)
1022 except:
1023 if _query_logging_verbosity == 1:
1024 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows')
1025
1026 if curs.description is not None:
1027 if curs.rowcount > 0:
1028 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?')
1029 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description))
1030
1031
1032 commit()
1033 close_cursor()
1034 close_conn()
1035
1036 if data is None: status = 1
1037 else: status = data
1038 if return_err_msg: return (status, '')
1039 return status
1040
1041 -def run_ro_query(link_obj = None, aQuery = None, get_col_idx = False, *args):
1042 """Runs a read-only query.
1043
1044 - link_obj can be a service name, connection or cursor object
1045
1046 - return status:
1047 - return data if get_col_idx is False
1048 - return (data, idx) if get_col_idx is True
1049
1050 - if query fails: data is None
1051 - if query is not a row-returning SQL statement: data is None
1052
1053 - data is a list of tuples [(w,x,y,z), (a,b,c,d), ...] where each tuple is a table row
1054 - idx is a map of column name to their position in the row tuples
1055 e.g. { 'name': 3, 'id':0, 'job_description': 2, 'location':1 }
1056
1057 usage: e.g. data[0][idx['name']] would return z from [(w,x,y,z ),(a,b,c,d)]
1058 """
1059
1060 if link_obj is None:
1061 raise TypeError, 'gmPG.run_ro_query(): link_obj must be of type service name, connection or cursor'
1062 if aQuery is None:
1063 raise TypeError, 'gmPG.run_ro_query(): forgot to pass in aQuery'
1064
1065 close_cursor = noop
1066 close_conn = noop
1067
1068 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'):
1069 curs = link_obj
1070
1071 elif (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')):
1072 curs = link_obj.cursor()
1073 close_cursor = curs.close
1074
1075 else:
1076 pool = ConnectionPool()
1077 conn = pool.GetConnection(link_obj, readonly = 1)
1078 if conn is None:
1079 _log.Log(gmLog.lErr, 'cannot get connection to service [%s]' % link_obj)
1080 if not get_col_idx:
1081 return None
1082 else:
1083 return None, None
1084 curs = conn.cursor()
1085 close_cursor = curs.close
1086 close_conn = pool.ReleaseConnection
1087
1088
1089 try:
1090 curs.execute(aQuery, *args)
1091 global last_ro_cursor_desc
1092 last_ro_cursor_desc = curs.description
1093 except:
1094 _log.LogException("query >>>%s<<< with args >>>%s<<< failed on link [%s]" % (aQuery[:250], str(args)[:250], link_obj), sys.exc_info(), verbose = _query_logging_verbosity)
1095 __log_PG_settings(curs)
1096 close_cursor()
1097 close_conn(link_obj)
1098 if not get_col_idx:
1099 return None
1100 else:
1101 return None, None
1102
1103
1104
1105 if curs.description is None:
1106 data = None
1107 _log.Log(gmLog.lErr, 'query did not return rows')
1108 else:
1109 try:
1110 data = curs.fetchall()
1111 except:
1112 _log.LogException('cursor.fetchall() failed on link [%s]' % link_obj, sys.exc_info(), verbose = _query_logging_verbosity)
1113 close_cursor()
1114 close_conn(link_obj)
1115 if not get_col_idx:
1116 return None
1117 else:
1118 return None, None
1119
1120
1121 close_conn(link_obj)
1122 if get_col_idx:
1123 col_idx = get_col_indices(curs)
1124 close_cursor()
1125 return data, col_idx
1126 else:
1127 close_cursor()
1128 return data
1129
1130
1132
1133 if aCursor is None:
1134 _log.Log(gmLog.lErr, 'need cursor to get column indices')
1135 return None
1136 if aCursor.description is None:
1137 _log.Log(gmLog.lErr, 'no result description available: cursor unused or last query did not select rows')
1138 return None
1139 col_indices = {}
1140 col_index = 0
1141 for col_desc in aCursor.description:
1142 col_indices[col_desc[0]] = col_index
1143 col_index += 1
1144 return col_indices
1145
1146
1147
1149
1150 if aCursor is None:
1151 _log.Log(gmLog.lErr, 'need cursor to determine primary key')
1152 return None
1153 if aTable is None:
1154 _log.Log(gmLog.lErr, 'need table name for which to determine primary key')
1155
1156 if not run_query(aCursor, None, query_pkey_name, aTable):
1157 _log.Log(gmLog.lErr, 'cannot determine primary key')
1158 return -1
1159 result = aCursor.fetchone()
1160 if result is None:
1161 return None
1162 return result[0]
1163
1165 """Returns a dictionary of referenced foreign keys.
1166
1167 key = column name of this table
1168 value = (referenced table name, referenced column name) tuple
1169 """
1170 manage_connection = 0
1171 close_cursor = 1
1172
1173 if hasattr(source, 'fetchone') and hasattr(source, 'description'):
1174 close_cursor = 0
1175 curs = source
1176
1177 elif (hasattr(source, 'commit') and hasattr(source, 'cursor')):
1178 curs = source.cursor()
1179
1180 else:
1181 manage_connection = 1
1182 pool = ConnectionPool()
1183 conn = pool.GetConnection(source)
1184 if conn is None:
1185 _log.Log(gmLog.lErr, 'cannot get fkey names on table [%s] from source [%s]' % (table, source))
1186 return None
1187 curs = conn.cursor()
1188
1189 if not run_query(curs, None, query_fkey_names, table):
1190 if close_cursor:
1191 curs.close()
1192 if manage_connection:
1193 pool.ReleaseConnection(source)
1194 _log.Log(gmLog.lErr, 'cannot get foreign keys on table [%s] from source [%s]' % (table, source))
1195 return None
1196
1197 fks = curs.fetchall()
1198 if close_cursor:
1199 curs.close()
1200 if manage_connection:
1201 pool.ReleaseConnection(source)
1202
1203 references = {}
1204 for fk in fks:
1205 fkname, src_table, target_table, tmp, src_col, target_col, tmp = string.split(fk[0], '\x00')
1206 references[src_col] = (target_table, target_col)
1207
1208 return references
1209
1210 -def add_housekeeping_todo(
1211 reporter='$RCSfile: gmPG.py,v $ $Revision: 1.90 $',
1212 receiver='DEFAULT',
1213 problem='lazy programmer',
1214 solution='lazy programmer',
1215 context='lazy programmer',
1216 category='lazy programmer'
1217 ):
1218 queries = []
1219 cmd = "insert into housekeeping_todo (reported_by, reported_to, problem, solution, context, category) values (%s, %s, %s, %s, %s, %s)"
1220 queries.append((cmd, [reporter, receiver, problem, solution, context, category]))
1221 cmd = "select currval('housekeeping_todo_pk_seq')"
1222 queries.append((cmd, []))
1223 result, err = run_commit('historica', queries, 1)
1224 if result is None:
1225 _log.Log(gmLog.lErr, err)
1226 return (None, err)
1227 return (1, result[0][0])
1228
1229
1230
1231
1232