Package Gnumed :: Package pycommon :: Module gmPG
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmPG

   1   
   2  # This remains for documentation only. 
   3  raise ImportError('This module is deprecated. Use gmPG2.py.') 
   4   
   5   
   6   
   7   
   8   
   9  """Broker for PostgreSQL distributed backend connections. 
  10   
  11  @copyright: author 
  12   
  13  TODO: iterator/generator batch fetching: 
  14          - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad 
  15          - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4" 
  16   
  17  winner: 
  18  def resultset_functional_batchgenerator(cursor, size=100): 
  19          for results in iter(lambda: cursor.fetchmany(size), []): 
  20                  for rec in results: 
  21                          yield rec 
  22  """ 
  23  # ======================================================================= 
  24  # $Source: /home/ncq/Projekte/cvs2git/vcs-mirror/gnumed/gnumed/client/pycommon/gmPG.py,v $ 
  25  __version__ = "$Revision: 1.90 $" 
  26  __author__  = "H.Herb <hherb@gnumed.net>, I.Haywood <i.haywood@ugrad.unimelb.edu.au>, K.Hilbert <Karsten.Hilbert@gmx.net>" 
  27  __license__ = 'GPL v2 or later (details at http://www.gnu.org)' 
  28   
  29  print "gmPG phased out, please replace with gmPG2" 
  30   
  31  import sys 
  32  sys.exit 
  33   
  34  _query_logging_verbosity = 1 
  35   
  36  # check whether this adapter module suits our needs 
  37  assert(float(dbapi.apilevel) >= 2.0) 
  38  assert(dbapi.threadsafety > 0) 
  39  assert(dbapi.paramstyle == 'pyformat') 
  40   
  41  _listener_api = None 
  42   
  43  # default encoding for connections 
  44  _default_client_encoding = {'wire': None, 'string': None} 
  45   
  46  # default time zone for connections 
  47  # OR: mxDT.now().gmtoffset() 
  48  if time.daylight: 
  49          tz = time.altzone 
  50  else: 
  51          tz = time.timezone 
  52  # do some magic to convert Python's timezone to a valid ISO timezone 
  53  # is this safe or will it return things like 13.5 hours ? 
  54  _default_client_timezone = "%+.1f" % (-tz / 3600.0) 
  55   
  56  _serialize_failure = "serialize access due to concurrent update" 
  57   
  58  #====================================================================== 
  59  # a bunch of useful queries 
  60  #---------------------------------------------------------------------- 
  61  QTablePrimaryKeyIndex = """ 
  62  SELECT 
  63          indkey 
  64  FROM 
  65          pg_index 
  66  WHERE 
  67          indrelid = 
  68          (SELECT oid FROM pg_class WHERE relname = '%s'); 
  69  """ 
  70   
  71  query_pkey_name = """ 
  72  SELECT 
  73          pga.attname 
  74  FROM 
  75          (pg_attribute pga inner join pg_index pgi on (pga.attrelid=pgi.indrelid)) 
  76  WHERE 
  77          pga.attnum=pgi.indkey[0] 
  78                  and 
  79          pgi.indisprimary is true 
  80                  and 
  81          pga.attrelid=(SELECT oid FROM pg_class WHERE relname = %s)""" 
  82   
  83  query_fkey_names = """ 
  84  select tgargs from pg_trigger where 
  85          tgname like 'RI%%' 
  86                  and 
  87          tgrelid = ( 
  88                  select oid from pg_class where relname=%s 
  89          ) 
  90  """ 
  91   
  92  # get columns and data types for a given table 
  93  query_table_col_defs = """select 
  94          cols.column_name, 
  95          cols.udt_name 
  96  from 
  97          information_schema.columns cols 
  98  where 
  99          cols.table_schema = %s 
 100                  and 
 101          cols.table_name = %s 
 102  order by 
 103          cols.ordinal_position""" 
 104   
 105  query_table_attributes = """select 
 106          cols.column_name 
 107  from 
 108          information_schema.columns cols 
 109  where 
 110          cols.table_schema = %s 
 111                  and 
 112          cols.table_name = %s 
 113  order by 
 114          cols.ordinal_position""" 
 115   
 116  query_child_tables = """ 
 117  select 
 118          pgn.nspname as namespace, 
 119          pgc.relname as table 
 120  from 
 121          pg_namespace pgn, 
 122          pg_class pgc 
 123  where 
 124          pgc.relnamespace = pgn.oid 
 125                  and 
 126          pgc.oid in ( 
 127                  select inhrelid from pg_inherits where inhparent = ( 
 128                          select oid from pg_class where 
 129                                  relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and 
 130                                  relname = %(table)s 
 131                  ) 
 132          )""" 
 133   
 134   
 135  # a handy return to dbapi simplicity 
 136  last_ro_cursor_desc = None 
 137   
 138  #====================================================================== 
139 -class ConnectionPool:
140 "maintains a static dictionary of available database connections" 141 142 # cached read-only connection objects 143 __ro_conns = {} 144 # maps service names to physical databases 145 __service2db_map = {} 146 # connections in use per service (for reference counting) 147 __conn_use_count = {} 148 # variable used to check whether a first connection has been initialized yet or not 149 __is_connected = None 150 # maps backend listening threads to database ids 151 __listeners = {} 152 # gmLoginInfo.LoginInfo instance 153 __login = None 154 #-----------------------------
155 - def __init__(self, login=None, encoding=None):
156 """parameter login is of type gmLoginInfo.LoginInfo""" 157 # if login data is given: re-establish connections 158 if login is not None: 159 self.__disconnect() 160 if ConnectionPool.__is_connected is None: 161 # CAREFUL: this affects the whole connection 162 dbapi.fetchReturnsList = True 163 ConnectionPool.__is_connected = self.__setup_default_ro_conns(login=login, encoding=encoding)
164 #-----------------------------
165 - def __del__(self):
166 pass
167 # NOTE: do not kill listeners here which would mean to 168 # kill them when we throw away *any* ConnectionPool 169 # instance - not what we want 170 #----------------------------- 171 # connection API 172 #-----------------------------
173 - def GetConnection(self, service="default", readonly=1, encoding=None, extra_verbose=None):
174 """Get a connection.""" 175 176 logininfo = self.GetLoginInfoFor(service) 177 178 # either get a cached read-only connection 179 if readonly: 180 if ConnectionPool.__ro_conns.has_key(service): 181 try: 182 ConnectionPool.__conn_use_count[service] += 1 183 except KeyError: 184 ConnectionPool.__conn_use_count[service] = 1 185 conn = ConnectionPool.__ro_conns[service] 186 else: 187 try: 188 ConnectionPool.__conn_use_count['default'] += 1 189 except KeyError: 190 ConnectionPool.__conn_use_count['default'] = 1 191 conn = ConnectionPool.__ro_conns['default'] 192 # or a brand-new read-write connection 193 else: 194 _log.Log(gmLog.lData, "requesting RW connection to service [%s]" % service) 195 conn = self.__pgconnect(logininfo, readonly = 0, encoding = encoding) 196 if conn is None: 197 return None 198 199 if extra_verbose: 200 conn.conn.toggleShowQuery 201 202 return conn
203 #-----------------------------
204 - def ReleaseConnection(self, service):
205 """decrease reference counter of active connection""" 206 if ConnectionPool.__ro_conns.has_key(service): 207 try: 208 ConnectionPool.__conn_use_count[service] -= 1 209 except: 210 ConnectionPool.__conn_use_count[service] = 0 211 else: 212 try: 213 ConnectionPool.__conn_use_count['default'] -= 1 214 except: 215 ConnectionPool.__conn_use_count['default'] = 0
216 #-----------------------------
217 - def Connected(self):
219 #-----------------------------
220 - def get_connection_for_user(self, user=None, password=None, service="default", encoding=None, extra_verbose=None):
221 """Get a connection for a given user. 222 223 This will return a connection just as GetConnection() would 224 except that the user to be used for authentication can be 225 specified. All the other parameters are going to be the 226 same, IOW it will connect to the same server, port and database 227 as any other connection obtained through this broker. 228 229 You will have to specify the password, of course, if it 230 is needed for PostgreSQL authentication. 231 232 This will always return a read-write connection. 233 """ 234 if user is None: 235 _log.Log(gmLog.lErr, 'user must be given') 236 raise ValueError, 'gmPG.py::%s.get_connection_for_user(): user name must be given' % self.__class__.__name__ 237 238 logininfo = self.GetLoginInfoFor(service) 239 logininfo.SetUser(user=user) 240 logininfo.SetPassword(passwd=password) 241 242 _log.Log(gmLog.lData, "requesting RW connection to service [%s]" % service) 243 conn = self.__pgconnect(logininfo, readonly = 0, encoding = encoding) 244 if conn is None: 245 return None 246 247 if extra_verbose: 248 conn.conn.toggleShowQuery 249 250 return conn
251 #----------------------------- 252 # notification API 253 #-----------------------------
254 - def Listen(self, service, signal, callback):
255 """Listen to 'signal' from backend in an asynchronous thread. 256 257 If 'signal' is received from database 'service', activate 258 the 'callback' function""" 259 # FIXME: error handling 260 261 # lazy import of gmBackendListener 262 if _listener_api is None: 263 if not _import_listener_engine(): 264 _log.Log(gmLog.lErr, 'cannot load backend listener code') 265 return None 266 267 # get physical database for service 268 try: 269 backend = ConnectionPool.__service2db_map[service] 270 except KeyError: 271 backend = 0 272 _log.Log(gmLog.lData, "connecting notification [%s] from service [%s] (id %s) with callback %s" % (signal, service, backend, callback)) 273 # start thread if not listening yet, 274 # but only one per physical database 275 if backend not in ConnectionPool.__listeners.keys(): 276 auth = self.GetLoginInfoFor(service) 277 listener = _listener_api.BackendListener( 278 service, 279 auth.GetDatabase(), 280 auth.GetUser(), 281 auth.GetPassword(), 282 auth.GetHost(), 283 int(auth.GetPort()) 284 ) 285 ConnectionPool.__listeners[backend] = listener 286 # actually start listening 287 listener = ConnectionPool.__listeners[backend] 288 listener.register_callback(signal, callback) 289 return 1
290 #-----------------------------
291 - def Unlisten(self, service, signal, callback):
292 # get physical database for service 293 try: 294 backend = ConnectionPool.__service2db_map[service] 295 except KeyError: 296 backend = 0 297 _log.Log(gmLog.lData, "disconnecting notification [%s] from service [%s] (id %s) from callback %s" % (signal, service, backend, callback)) 298 if backend not in ConnectionPool.__listeners.keys(): 299 return 1 300 listener = ConnectionPool.__listeners[backend] 301 listener.unregister_callback(signal, callback)
302 #-----------------------------
303 - def StopListener(self, service):
304 try: 305 backend = self.__service2db_map[service] 306 except KeyError: 307 _log.Log(gmLog.lWarn, 'cannot stop listener on backend') 308 return None 309 try: 310 ConnectionPool.__listeners[backend].stop_thread() 311 del ConnectionPool.__listeners[backend] 312 except: 313 _log.LogException('cannot stop listener on backend [%s]' % backend, sys.exc_info(), verbose = 0) 314 return None 315 return 1
316 #-----------------------------
317 - def StopListeners(self):
318 for backend in ConnectionPool.__listeners.keys(): 319 try: 320 ConnectionPool.__listeners[backend].stop_thread() 321 del ConnectionPool.__listeners[backend] 322 except: 323 _log.LogException('cannot stop listener on backend [%s]' % backend, sys.exc_info(), verbose = 0) 324 return 1
325 #----------------------------- 326 # misc API 327 #-----------------------------
328 - def GetAvailableServices(self):
329 """list all distributed services available on this system 330 (according to configuration database)""" 331 return ConnectionPool.__ro_conns.keys()
332 #-----------------------------
333 - def GetLoginInfoFor(self, service, login = None):
334 """return login information for a particular service""" 335 if login is None: 336 dblogin = ConnectionPool.__login 337 else: 338 dblogin = copy.deepcopy(login) 339 # if service not mapped, return default login information 340 try: 341 srvc_id = ConnectionPool.__service2db_map[service] 342 except KeyError: 343 return dblogin 344 # a service in the default database 345 if srvc_id == 0: 346 return dblogin 347 # actually fetch parameters for db where service 348 # is located from config DB 349 cfg_db = ConnectionPool.__ro_conns['default'] 350 cursor = cfg_db.cursor() 351 cmd = "select name, host, port from cfg.db where pk=%s" 352 if not run_query(cursor, None, cmd, srvc_id): 353 _log.Log(gmLog.lPanic, 'cannot get login info for service [%s] with id [%s] from config database' % (service, srvc_id)) 354 _log.Log(gmLog.lPanic, 'make sure your service-to-database mappings are properly configured') 355 _log.Log(gmLog.lWarn, 'trying to make do with default login parameters') 356 return dblogin 357 auth_data = cursor.fetchone() 358 idx = get_col_indices(cursor) 359 cursor.close() 360 # substitute values into default login data 361 try: # db name 362 dblogin.SetDatabase(string.strip(auth_data[idx['name']])) 363 except: pass 364 try: # host name 365 dblogin.SetHost(string.strip(auth_data[idx['host']])) 366 except: pass 367 try: # port 368 dblogin.SetPort(auth_data[idx['port']]) 369 except: pass 370 # and return what we thus got - which may very well be identical to the default login ... 371 return dblogin
372 #----------------------------- 373 # private methods 374 #-----------------------------
375 - def __setup_default_ro_conns(self, login=None, encoding=None):
376 """Initialize connections to all servers.""" 377 if login is None and ConnectionPool.__is_connected is None: 378 try: 379 login = request_login_params() 380 except: 381 _log.LogException("Exception: Cannot connect to databases without login information !", sys.exc_info(), verbose=1) 382 raise gmExceptions.ConnectionError("Can't connect to database without login information!") 383 384 _log.Log(gmLog.lData, login.GetInfoStr()) 385 ConnectionPool.__login = login 386 387 # connect to the configuration server 388 cfg_db = self.__pgconnect(login, readonly=1, encoding=encoding) 389 if cfg_db is None: 390 raise gmExceptions.ConnectionError, _('Cannot connect to configuration database with:\n\n[%s]') % login.GetInfoStr() 391 392 # this is the default gnumed server now 393 ConnectionPool.__ro_conns['default'] = cfg_db 394 cursor = cfg_db.cursor() 395 # document DB version 396 cursor.execute("select version()") 397 _log.Log(gmLog.lInfo, 'service [default/config] running on [%s]' % cursor.fetchone()[0]) 398 # preload all services with database pk 0 (default) 399 cmd = "select name from cfg.distributed_db" 400 if not run_query(cursor, None, cmd): 401 cursor.close() 402 raise gmExceptions.ConnectionError("cannot load service names from configuration database") 403 services = cursor.fetchall() 404 for service in services: 405 ConnectionPool.__service2db_map[service[0]] = 0 406 407 # establish connections to all servers we need 408 # according to configuration database 409 cmd = "select * from cfg.config where profile=%s" 410 if not run_query(cursor, None, cmd, login.GetProfile()): 411 cursor.close() 412 raise gmExceptions.ConnectionError("cannot load user profile [%s] from database" % login.GetProfile()) 413 databases = cursor.fetchall() 414 dbidx = get_col_indices(cursor) 415 416 # for all configuration entries that match given user and profile 417 for db in databases: 418 # - get symbolic name of distributed service 419 cursor.execute("select name from cfg.distributed_db where pk=%d" % db[dbidx['ddb']]) 420 service = string.strip(cursor.fetchone()[0]) 421 # - map service name to id of real database 422 _log.Log(gmLog.lData, "mapping service [%s] to DB ID [%s]" % (service, db[dbidx['db']])) 423 ConnectionPool.__service2db_map[service] = db[dbidx['db']] 424 # - init ref counter 425 ConnectionPool.__conn_use_count[service] = 0 426 dblogin = self.GetLoginInfoFor(service, login) 427 # - update 'Database Broker' dictionary 428 conn = self.__pgconnect(dblogin, readonly=1, encoding=encoding) 429 if conn is None: 430 raise gmExceptions.ConnectionError, _('Cannot connect to database with:\n\n[%s]') % login.GetInfoStr() 431 ConnectionPool.__ro_conns[service] = conn 432 # - document DB version 433 cursor.execute("select version()") 434 _log.Log(gmLog.lInfo, 'service [%s] running on [%s]' % (service, cursor.fetchone()[0])) 435 cursor.close() 436 ConnectionPool.__is_connected = 1 437 return ConnectionPool.__is_connected
438 #-----------------------------
439 - def __pgconnect(self, login, readonly=1, encoding=None):
440 """Connect to a postgres backend as specified by login object. 441 442 - returns a connection object 443 - encoding works like this: 444 - encoding specified in the call to __pgconnect() overrides 445 - encoding set by a call to gmPG.set_default_encoding() overrides 446 - encoding taken from Python string encoding 447 - wire_encoding and string_encoding must essentially just be different 448 names for one and the same (IOW entirely compatible) encodings, such 449 as "win1250" and "cp1250" 450 """ 451 dsn = "" 452 hostport = "" 453 dsn = login.GetDBAPI_DSN() 454 hostport = "0" 455 456 if encoding is None: 457 encoding = _default_client_encoding 458 459 # encoding a Unicode string with this encoding must 460 # yield a byte string encoded such that it can be decoded 461 # safely by wire_encoding 462 string_encoding = encoding['string'] 463 if string_encoding is None: 464 string_encoding = _default_client_encoding['string'] 465 if string_encoding is None: 466 # string_encoding = sys.getdefaultencoding() 467 string_encoding = locale.getlocale()[1] 468 _log.Log(gmLog.lWarn, 'client encoding not specified, this may lead to data corruption in some cases') 469 _log.Log(gmLog.lWarn, 'therefore the string encoding currently set in the active locale is used: [%s]' % string_encoding) 470 _log.Log(gmLog.lWarn, 'for this to have any chance to work the application MUST have called locale.setlocale() before') 471 _log.Log(gmLog.lInfo, 'using string encoding [%s] to encode Unicode strings for transmission to the database' % string_encoding) 472 473 # Python does not necessarily have to know this encoding by name 474 # but it must know an equivalent encoding which guarantees roundtrip 475 # equality (set that via string_encoding) 476 wire_encoding = encoding['wire'] 477 if wire_encoding is None: 478 wire_encoding = _default_client_encoding['wire'] 479 if wire_encoding is None: 480 wire_encoding = string_encoding 481 if wire_encoding is None: 482 raise ValueError, '<wire_encoding> cannot be None' 483 484 try: 485 # FIXME: eventually use UTF or UTF8 for READONLY connections _only_ 486 conn = dbapi.connect(dsn=dsn, client_encoding=(string_encoding, 'strict'), unicode_results=1) 487 except StandardError: 488 _log.LogException("database connection failed: DSN = [%s], host:port = [%s]" % (dsn, hostport), sys.exc_info(), verbose = 1) 489 return None 490 491 # set the default characteristics of our sessions 492 curs = conn.cursor() 493 494 # - client encoding 495 cmd = "set client_encoding to '%s'" % wire_encoding 496 try: 497 curs.execute(cmd) 498 except: 499 curs.close() 500 conn.close() 501 _log.Log(gmLog.lErr, 'query [%s]' % cmd) 502 _log.LogException ( 503 'cannot set string-on-the-wire client_encoding on connection to [%s], this would likely lead to data corruption' % wire_encoding, 504 sys.exc_info(), 505 verbose = _query_logging_verbosity 506 ) 507 raise 508 _log.Log(gmLog.lData, 'string-on-the-wire client_encoding set to [%s]' % wire_encoding) 509 510 # - client time zone 511 # cmd = "set session time zone interval '%s'" % _default_client_timezone 512 cmd = "set time zone '%s'" % _default_client_timezone 513 if not run_query(curs, None, cmd): 514 _log.Log(gmLog.lErr, 'cannot set client time zone to [%s]' % _default_client_timezone) 515 _log.Log(gmLog.lWarn, 'not setting this will lead to incorrect dates/times') 516 else: 517 _log.Log (gmLog.lData, 'time zone set to [%s]' % _default_client_timezone) 518 519 # - datestyle 520 # FIXME: add DMY/YMD handling 521 cmd = "set datestyle to 'ISO'" 522 if not run_query(curs, None, cmd): 523 _log.Log(gmLog.lErr, 'cannot set client date style to ISO') 524 _log.Log(gmLog.lWarn, 'you better use other means to make your server delivers valid ISO timestamps with time zone') 525 526 # - transaction isolation level 527 if readonly: 528 isolation_level = 'READ COMMITTED' 529 else: 530 isolation_level = 'SERIALIZABLE' 531 cmd = 'set session characteristics as transaction isolation level %s' % isolation_level 532 if not run_query(curs, None, cmd): 533 curs.close() 534 conn.close() 535 _log.Log(gmLog.lErr, 'cannot set connection characteristics to [%s]' % isolation_level) 536 return None 537 538 # - access mode 539 if readonly: 540 access_mode = 'READ ONLY' 541 else: 542 access_mode = 'READ WRITE' 543 _log.Log(gmLog.lData, "setting session to [%s] for %s@%s:%s" % (access_mode, login.GetUser(), login.GetHost(), login.GetDatabase())) 544 cmd = 'set session characteristics as transaction %s' % access_mode 545 if not run_query(curs, 0, cmd): 546 _log.Log(gmLog.lErr, 'cannot set connection characteristics to [%s]' % access_mode) 547 curs.close() 548 conn.close() 549 return None 550 551 conn.commit() 552 curs.close() 553 return conn
554 #-----------------------------
555 - def __disconnect(self, force_it=0):
556 """safe disconnect (respecting possibly active connections) unless the force flag is set""" 557 # are we connected at all? 558 if ConnectionPool.__is_connected is None: 559 # just in case 560 ConnectionPool.__ro_conns.clear() 561 return 562 # stop all background threads 563 for backend in ConnectionPool.__listeners.keys(): 564 ConnectionPool.__listeners[backend].stop_thread() 565 del ConnectionPool.__listeners[backend] 566 # disconnect from all databases 567 for key in ConnectionPool.__ro_conns.keys(): 568 # check whether this connection might still be in use ... 569 if ConnectionPool.__conn_use_count[key] > 0 : 570 # unless we are really mean 571 if force_it == 0: 572 # let the end user know that shit is happening 573 raise gmExceptions.ConnectionError, "Attempting to close a database connection that is still in use" 574 else: 575 # close the connection 576 ConnectionPool.__ro_conns[key].close() 577 578 # clear the dictionary (would close all connections anyway) 579 ConnectionPool.__ro_conns.clear() 580 ConnectionPool.__is_connected = None
581 582 #--------------------------------------------------- 583 # database helper functions 584 #---------------------------------------------------
585 -def fieldNames(cursor):
586 "returns the attribute names of the fetched rows in natural sequence as a list" 587 names=[] 588 for d in cursor.description: 589 names.append(d[0]) 590 return names
591 #---------------------------------------------------
592 -def run_query(aCursor=None, verbosity=None, aQuery=None, *args):
593 # sanity checks 594 if aCursor is None: 595 _log.Log(gmLog.lErr, 'need cursor to run query') 596 return None 597 if aQuery is None: 598 _log.Log(gmLog.lErr, 'need query to run it') 599 return None 600 if verbosity is None: 601 verbosity = _query_logging_verbosity 602 603 # t1 = time.time() 604 try: 605 aCursor.execute(aQuery, *args) 606 except: 607 _log.LogException("query >>>%s<<< with args >>>%s<<< failed" % (aQuery, args), sys.exc_info(), verbose = verbosity) 608 return None 609 # t2 = time.time() 610 # print t2-t1, aQuery 611 return 1
612 #---------------------------------------------------
613 -def run_commit2(link_obj=None, queries=None, end_tx=False, max_tries=1, extra_verbose=False, get_col_idx = False):
614 """Convenience function for running a transaction 615 that is supposed to get committed. 616 617 <link_obj> 618 can be either: 619 - a cursor 620 - a connection 621 - a service name 622 623 <queries> 624 is a list of (query, [args]) tuples to be 625 executed as a single transaction, the last 626 query may usefully return rows (such as a 627 "select currval('some_sequence')" statement) 628 629 <end_tx> 630 - controls whether the transaction is finalized (eg. 631 committed/rolled back) or not, this allows the 632 call to run_commit2() to be part of a framing 633 transaction 634 - if <link_obj> is a service name the transaction is 635 always finalized regardless of what <end_tx> says 636 - if link_obj is a connection then <end_tx> will 637 default to False unless it is explicitly set to 638 True which is taken to mean "yes, you do have full 639 control over the transaction" in which case the 640 transaction is properly finalized 641 642 <max_tries> 643 - controls the number of times a transaction is retried 644 after a concurrency error 645 - note that *all* <queries> are rerun if a concurrency 646 error occurrs 647 - max_tries is honored if and only if link_obj is a service 648 name such that we have full control over the transaction 649 650 <get_col_idx> 651 - if true, the returned data will include a dictionary 652 mapping field names to column positions 653 - if false, the returned data returns an empty dict 654 655 method result: 656 - returns a tuple (status, data) 657 - <status>: 658 * True - if all queries succeeded (also if there were 0 queries) 659 * False - if *any* error occurred 660 - <data> if <status> is True: 661 * (None, {}) if last query did not return rows 662 * ("fetchall() result", <index>) if last query returned any rows 663 * for <index> see <get_col_idx> 664 - <data> if <status> is False: 665 * a tuple (error, message) where <error> can be: 666 * 1: unspecified error 667 * 2: concurrency error 668 * 3: constraint violation (non-primary key) 669 * 4: access violation 670 """ 671 # sanity checks 672 if queries is None: 673 return (False, (1, 'forgot to pass in queries')) 674 if len(queries) == 0: 675 return (True, 'no queries to execute') 676 677 # check link_obj 678 # is it a cursor ? 679 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'): 680 return __commit2cursor(cursor=link_obj, queries=queries, extra_verbose=extra_verbose, get_col_idx=get_col_idx) 681 # is it a connection ? 682 if (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')): 683 return __commit2conn(conn=link_obj, queries=queries, end_tx=end_tx, extra_verbose=extra_verbose, get_col_idx=get_col_idx) 684 # take it to be a service name then 685 return __commit2service(service=link_obj, queries=queries, max_tries=max_tries, extra_verbose=extra_verbose, get_col_idx=get_col_idx)
686 #---------------------------------------------------
687 -def __commit2service(service=None, queries=None, max_tries=1, extra_verbose=False, get_col_idx=False):
688 # sanity checks 689 try: int(max_tries) 690 except ValueEror: max_tries = 1 691 if max_tries > 4: 692 max_tries = 4 693 if max_tries < 1: 694 max_tries = 1 695 # get cursor 696 pool = ConnectionPool() 697 conn = pool.GetConnection(str(service), readonly = 0) 698 if conn is None: 699 msg = 'cannot connect to service [%s]' 700 _log.Log(gmLog.lErr, msg % service) 701 return (False, (1, _(msg) % service)) 702 if extra_verbose: 703 conn.conn.toggleShowQuery 704 curs = conn.cursor() 705 for attempt in range(0, max_tries): 706 if extra_verbose: 707 _log.Log(gmLog.lData, 'attempt %s' % attempt) 708 # run queries 709 for query, args in queries: 710 if extra_verbose: 711 t1 = time.time() 712 try: 713 curs.execute(query, *args) 714 # FIXME: be more specific in exception catching 715 except: 716 if extra_verbose: 717 duration = time.time() - t1 718 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration) 719 conn.rollback() 720 exc_info = sys.exc_info() 721 typ, val, tb = exc_info 722 if str(val).find(_serialize_failure) > 0: 723 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update') 724 if attempt < max_tries: 725 # jump to next full attempt 726 time.sleep(0.1) 727 continue 728 curs.close() 729 conn.close() 730 return (False, (2, 'l')) 731 # FIXME: handle more types of errors 732 _log.Log(gmLog.lErr, 'query: %s' % query[:2048]) 733 try: 734 _log.Log(gmLog.lErr, 'argument: %s' % str(args)[:2048]) 735 except MemoryError: 736 pass 737 _log.LogException("query failed on link [%s]" % service, exc_info) 738 if extra_verbose: 739 __log_PG_settings(curs) 740 curs.close() 741 conn.close() 742 tmp = str(val).replace('ERROR:', '') 743 tmp = tmp.replace('ExecAppend:', '') 744 tmp = tmp.strip() 745 return (False, (1, _('SQL: %s') % tmp)) 746 # apparently succeeded 747 if extra_verbose: 748 duration = time.time() - t1 749 _log.Log(gmLog.lData, 'query: %s' % query[:2048]) 750 try: 751 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048]) 752 except MemoryError: 753 pass 754 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % service) 755 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration)) 756 # done with queries 757 break # out of retry loop 758 # done with attempt(s) 759 # did we get result rows in the last query ? 760 data = None 761 idx = {} 762 # now, the DB-API is ambigous about whether cursor.description 763 # and cursor.rowcount apply to the most recent query in a cursor 764 # (does this statement make any sense in the first place ?) or 765 # to the entire lifetime of said cursor, pyPgSQL thinks the 766 # latter, hence we need to catch exceptions when there's no 767 # data from the *last* query 768 try: 769 data = curs.fetchall() 770 except: 771 if extra_verbose: 772 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows') 773 # should be None if no rows were returned ... 774 if curs.description is not None: 775 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?') 776 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description)) 777 conn.commit() 778 if get_col_idx: 779 idx = get_col_indices(curs) 780 curs.close() 781 conn.close() 782 return (True, (data, idx))
783 #---------------------------------------------------
784 -def __commit2conn(conn=None, queries=None, end_tx=False, extra_verbose=False, get_col_idx=False):
785 if extra_verbose: 786 conn.conn.toggleShowQuery 787 788 # get cursor 789 curs = conn.cursor() 790 791 # run queries 792 for query, args in queries: 793 if extra_verbose: 794 t1 = time.time() 795 try: 796 curs.execute(query, *args) 797 except: 798 if extra_verbose: 799 duration = time.time() - t1 800 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration) 801 conn.rollback() 802 exc_info = sys.exc_info() 803 typ, val, tb = exc_info 804 if str(val).find(_serialize_failure) > 0: 805 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update') 806 curs.close() 807 if extra_verbose: 808 conn.conn.toggleShowQuery 809 return (False, (2, 'l')) 810 # FIXME: handle more types of errors 811 _log.Log(gmLog.lErr, 'query: %s' % query[:2048]) 812 try: 813 _log.Log(gmLog.lErr, 'args : %s' % str(args)[:2048]) 814 except MemoryError: 815 pass 816 _log.LogException("query failed on link [%s]" % conn, exc_info) 817 if extra_verbose: 818 __log_PG_settings(curs) 819 curs.close() 820 tmp = str(val).replace('ERROR:', '') 821 tmp = tmp.replace('ExecAppend:', '') 822 tmp = tmp.strip() 823 if extra_verbose: 824 conn.conn.toggleShowQuery 825 return (False, (1, _('SQL: %s') % tmp)) 826 # apparently succeeded 827 if extra_verbose: 828 duration = time.time() - t1 829 _log.Log(gmLog.lData, 'query: %s' % query[:2048]) 830 try: 831 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048]) 832 except MemoryError: 833 pass 834 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % conn) 835 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration)) 836 # done with queries 837 if extra_verbose: 838 conn.conn.toggleShowQuery 839 # did we get result rows in the last query ? 840 data = None 841 idx = {} 842 # now, the DB-API is ambigous about whether cursor.description 843 # and cursor.rowcount apply to the most recent query in a cursor 844 # (does this statement make any sense in the first place ?) or 845 # to the entire lifetime of said cursor, pyPgSQL thinks the 846 # latter, hence we need to catch exceptions when there's no 847 # data from the *last* query 848 try: 849 data = curs.fetchall() 850 except: 851 if extra_verbose: 852 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows') 853 # should be None if no rows were returned ... 854 if curs.description is not None: 855 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?') 856 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description)) 857 if end_tx: 858 conn.commit() 859 if get_col_idx: 860 idx = get_col_indices(curs) 861 curs.close() 862 return (True, (data, idx))
863 #---------------------------------------------------
864 -def __commit2cursor(cursor=None, queries=None, extra_verbose=False, get_col_idx=False):
865 # run queries 866 for query, args in queries: 867 if extra_verbose: 868 t1 = time.time() 869 try: 870 curs.execute(query, *args) 871 except: 872 if extra_verbose: 873 duration = time.time() - t1 874 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration) 875 exc_info = sys.exc_info() 876 typ, val, tb = exc_info 877 if str(val).find(_serialize_failure) > 0: 878 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update') 879 return (False, (2, 'l')) 880 # FIXME: handle more types of errors 881 _log.Log(gmLog.lErr, 'query: %s' % query[:2048]) 882 try: 883 _log.Log(gmLog.lErr, 'args : %s' % str(args)[:2048]) 884 except MemoryError: 885 pass 886 _log.LogException("query failed on link [%s]" % cursor, exc_info) 887 if extra_verbose: 888 __log_PG_settings(curs) 889 tmp = str(val).replace('ERROR:', '') 890 tmp = tmp.replace('ExecAppend:', '') 891 tmp = tmp.strip() 892 return (False, (1, _('SQL: %s') % tmp)) 893 # apparently succeeded 894 if extra_verbose: 895 duration = time.time() - t1 896 _log.Log(gmLog.lData, 'query: %s' % query[:2048]) 897 try: 898 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048]) 899 except MemoryError: 900 pass 901 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % cursor) 902 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration)) 903 904 # did we get result rows in the last query ? 905 data = None 906 idx = {} 907 # now, the DB-API is ambigous about whether cursor.description 908 # and cursor.rowcount apply to the most recent query in a cursor 909 # (does this statement make any sense in the first place ?) or 910 # to the entire lifetime of said cursor, pyPgSQL thinks the 911 # latter, hence we need to catch exceptions when there's no 912 # data from the *last* query 913 try: 914 data = curs.fetchall() 915 except: 916 if extra_verbose: 917 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows') 918 # should be None if no rows were returned ... 919 if curs.description is not None: 920 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?') 921 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description)) 922 if get_col_idx: 923 idx = get_col_indices(curs) 924 return (True, (data, idx))
925 #---------------------------------------------------
926 -def run_commit(link_obj = None, queries = None, return_err_msg = None):
927 """Convenience function for running a transaction 928 that is supposed to get committed. 929 930 - link_obj can be 931 - a cursor: rollback/commit must be done by the caller 932 - a connection: rollback/commit is handled 933 - a service name: rollback/commit is handled 934 935 - queries is a list of (query, [args]) tuples 936 - executed as a single transaction 937 938 - returns: 939 - a tuple (<value>, error) if return_err_msg is True 940 - a scalar <value> if return_err_msg is False 941 942 - <value> will be 943 - None: if any query failed 944 - 1: if all queries succeeded (also 0 queries) 945 - data: if the last query returned rows 946 """ 947 print "DEPRECATION WARNING: gmPG.run_commit() is deprecated, use run_commit2() instead" 948 949 # sanity checks 950 if link_obj is None: 951 raise TypeError, 'gmPG.run_commit(): link_obj must be of type service name, connection or cursor' 952 if queries is None: 953 raise TypeError, 'gmPG.run_commit(): forgot to pass in queries' 954 if len(queries) == 0: 955 _log.Log(gmLog.lWarn, 'no queries to execute ?!?') 956 if return_err_msg: 957 return (1, 'no queries to execute ?!?') 958 return 1 959 960 close_cursor = noop 961 close_conn = noop 962 commit = noop 963 rollback = noop 964 # is it a cursor ? 965 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'): 966 curs = link_obj 967 # is it a connection ? 968 elif (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')): 969 curs = link_obj.cursor() 970 close_cursor = curs.close 971 conn = link_obj 972 commit = link_obj.commit 973 rollback = link_obj.rollback 974 # take it to be a service name then 975 else: 976 pool = ConnectionPool() 977 conn = pool.GetConnection(link_obj, readonly = 0) 978 if conn is None: 979 _log.Log(gmLog.lErr, 'cannot connect to service [%s]' % link_obj) 980 if return_err_msg: 981 return (None, _('cannot connect to service [%s]') % link_obj) 982 return None 983 curs = conn.cursor() 984 close_cursor = curs.close 985 close_conn = conn.close 986 commit = conn.commit 987 rollback = conn.rollback 988 # run queries 989 for query, args in queries: 990 # t1 = time.time() 991 try: 992 curs.execute (query, *args) 993 except: 994 rollback() 995 exc_info = sys.exc_info() 996 _log.LogException ("RW query >>>%s<<< with args >>>%s<<< failed on link [%s]" % (query[:1024], str(args)[:1024], link_obj), exc_info, verbose = _query_logging_verbosity) 997 __log_PG_settings(curs) 998 close_cursor() 999 close_conn() 1000 if return_err_msg: 1001 typ, val, tb = exc_info 1002 tmp = string.replace(str(val), 'ERROR:', '') 1003 tmp = string.replace(tmp, 'ExecAppend:', '') 1004 tmp = string.strip(tmp) 1005 return (None, 'SQL: %s' % tmp) 1006 return None 1007 # t2 = time.time() 1008 # print t2-t1, query 1009 if _query_logging_verbosity == 1: 1010 _log.Log(gmLog.lData, '%s rows affected by >>>%s<<<' % (curs.rowcount, query)) 1011 # did we get result rows in the last query ? 1012 data = None 1013 # now, the DB-API is ambigous about whether cursor.description 1014 # and cursor.rowcount apply to the most recent query in a cursor 1015 # (does that statement make any sense ?!?) or to the entire lifetime 1016 # of said cursor, pyPgSQL thinks the latter, hence we need to catch 1017 # exceptions when there's no data from the *last* query 1018 try: 1019 data = curs.fetchall() 1020 if _query_logging_verbosity == 1: 1021 _log.Log(gmLog.lData, 'last query returned %s rows' % curs.rowcount) 1022 except: 1023 if _query_logging_verbosity == 1: 1024 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows') 1025 # something seems odd 1026 if curs.description is not None: 1027 if curs.rowcount > 0: 1028 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?') 1029 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description)) 1030 1031 # clean up 1032 commit() 1033 close_cursor() 1034 close_conn() 1035 1036 if data is None: status = 1 1037 else: status = data 1038 if return_err_msg: return (status, '') 1039 return status
1040 #---------------------------------------------------
1041 -def run_ro_query(link_obj = None, aQuery = None, get_col_idx = False, *args):
1042 """Runs a read-only query. 1043 1044 - link_obj can be a service name, connection or cursor object 1045 1046 - return status: 1047 - return data if get_col_idx is False 1048 - return (data, idx) if get_col_idx is True 1049 1050 - if query fails: data is None 1051 - if query is not a row-returning SQL statement: data is None 1052 1053 - data is a list of tuples [(w,x,y,z), (a,b,c,d), ...] where each tuple is a table row 1054 - idx is a map of column name to their position in the row tuples 1055 e.g. { 'name': 3, 'id':0, 'job_description': 2, 'location':1 } 1056 1057 usage: e.g. data[0][idx['name']] would return z from [(w,x,y,z ),(a,b,c,d)] 1058 """ 1059 # sanity checks 1060 if link_obj is None: 1061 raise TypeError, 'gmPG.run_ro_query(): link_obj must be of type service name, connection or cursor' 1062 if aQuery is None: 1063 raise TypeError, 'gmPG.run_ro_query(): forgot to pass in aQuery' 1064 1065 close_cursor = noop 1066 close_conn = noop 1067 # is it a cursor ? 1068 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'): 1069 curs = link_obj 1070 # is it a connection ? 1071 elif (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')): 1072 curs = link_obj.cursor() 1073 close_cursor = curs.close 1074 # take it to be a service name then 1075 else: 1076 pool = ConnectionPool() 1077 conn = pool.GetConnection(link_obj, readonly = 1) 1078 if conn is None: 1079 _log.Log(gmLog.lErr, 'cannot get connection to service [%s]' % link_obj) 1080 if not get_col_idx: 1081 return None 1082 else: 1083 return None, None 1084 curs = conn.cursor() 1085 close_cursor = curs.close 1086 close_conn = pool.ReleaseConnection 1087 # t1 = time.time() 1088 # run the query 1089 try: 1090 curs.execute(aQuery, *args) 1091 global last_ro_cursor_desc 1092 last_ro_cursor_desc = curs.description 1093 except: 1094 _log.LogException("query >>>%s<<< with args >>>%s<<< failed on link [%s]" % (aQuery[:250], str(args)[:250], link_obj), sys.exc_info(), verbose = _query_logging_verbosity) # this can fail on *large* args 1095 __log_PG_settings(curs) 1096 close_cursor() 1097 close_conn(link_obj) 1098 if not get_col_idx: 1099 return None 1100 else: 1101 return None, None 1102 # t2 = time.time() 1103 # print t2-t1, aQuery 1104 # and return the data, possibly including the column index 1105 if curs.description is None: 1106 data = None 1107 _log.Log(gmLog.lErr, 'query did not return rows') 1108 else: 1109 try: 1110 data = curs.fetchall() 1111 except: 1112 _log.LogException('cursor.fetchall() failed on link [%s]' % link_obj, sys.exc_info(), verbose = _query_logging_verbosity) 1113 close_cursor() 1114 close_conn(link_obj) 1115 if not get_col_idx: 1116 return None 1117 else: 1118 return None, None 1119 1120 # can "close" before closing cursor since it just decrements the ref counter 1121 close_conn(link_obj) 1122 if get_col_idx: 1123 col_idx = get_col_indices(curs) 1124 close_cursor() 1125 return data, col_idx 1126 else: 1127 close_cursor() 1128 return data
1129 #--------------------------------------------------- 1130 #---------------------------------------------------
1131 -def get_col_indices(aCursor = None):
1132 # sanity checks 1133 if aCursor is None: 1134 _log.Log(gmLog.lErr, 'need cursor to get column indices') 1135 return None 1136 if aCursor.description is None: 1137 _log.Log(gmLog.lErr, 'no result description available: cursor unused or last query did not select rows') 1138 return None 1139 col_indices = {} 1140 col_index = 0 1141 for col_desc in aCursor.description: 1142 col_indices[col_desc[0]] = col_index 1143 col_index += 1 1144 return col_indices
1145 #--------------------------------------------------- 1146 #--------------------------------------------------- 1147 #---------------------------------------------------
1148 -def get_pkey_name(aCursor = None, aTable = None):
1149 # sanity checks 1150 if aCursor is None: 1151 _log.Log(gmLog.lErr, 'need cursor to determine primary key') 1152 return None 1153 if aTable is None: 1154 _log.Log(gmLog.lErr, 'need table name for which to determine primary key') 1155 1156 if not run_query(aCursor, None, query_pkey_name, aTable): 1157 _log.Log(gmLog.lErr, 'cannot determine primary key') 1158 return -1 1159 result = aCursor.fetchone() 1160 if result is None: 1161 return None 1162 return result[0]
1163 #---------------------------------------------------
1164 -def get_fkey_defs(source, table):
1165 """Returns a dictionary of referenced foreign keys. 1166 1167 key = column name of this table 1168 value = (referenced table name, referenced column name) tuple 1169 """ 1170 manage_connection = 0 1171 close_cursor = 1 1172 # is it a cursor ? 1173 if hasattr(source, 'fetchone') and hasattr(source, 'description'): 1174 close_cursor = 0 1175 curs = source 1176 # is it a connection ? 1177 elif (hasattr(source, 'commit') and hasattr(source, 'cursor')): 1178 curs = source.cursor() 1179 # take it to be a service name then 1180 else: 1181 manage_connection = 1 1182 pool = ConnectionPool() 1183 conn = pool.GetConnection(source) 1184 if conn is None: 1185 _log.Log(gmLog.lErr, 'cannot get fkey names on table [%s] from source [%s]' % (table, source)) 1186 return None 1187 curs = conn.cursor() 1188 1189 if not run_query(curs, None, query_fkey_names, table): 1190 if close_cursor: 1191 curs.close() 1192 if manage_connection: 1193 pool.ReleaseConnection(source) 1194 _log.Log(gmLog.lErr, 'cannot get foreign keys on table [%s] from source [%s]' % (table, source)) 1195 return None 1196 1197 fks = curs.fetchall() 1198 if close_cursor: 1199 curs.close() 1200 if manage_connection: 1201 pool.ReleaseConnection(source) 1202 1203 references = {} 1204 for fk in fks: 1205 fkname, src_table, target_table, tmp, src_col, target_col, tmp = string.split(fk[0], '\x00') 1206 references[src_col] = (target_table, target_col) 1207 1208 return references
1209 #---------------------------------------------------
1210 -def add_housekeeping_todo( 1211 reporter='$RCSfile: gmPG.py,v $ $Revision: 1.90 $', 1212 receiver='DEFAULT', 1213 problem='lazy programmer', 1214 solution='lazy programmer', 1215 context='lazy programmer', 1216 category='lazy programmer' 1217 ):
1218 queries = [] 1219 cmd = "insert into housekeeping_todo (reported_by, reported_to, problem, solution, context, category) values (%s, %s, %s, %s, %s, %s)" 1220 queries.append((cmd, [reporter, receiver, problem, solution, context, category])) 1221 cmd = "select currval('housekeeping_todo_pk_seq')" 1222 queries.append((cmd, [])) 1223 result, err = run_commit('historica', queries, 1) 1224 if result is None: 1225 _log.Log(gmLog.lErr, err) 1226 return (None, err) 1227 return (1, result[0][0])
1228 #================================================================== 1229 #================================================================== 1230 # Main - unit testing 1231 #------------------------------------------------------------------ 1232